Move the native branch to master.
diff --git a/.gitignore b/.gitignore
index dff137a..a0c292d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,6 @@
.idea
-pkg
+go.mod
+go.sum
+vendor/
+coverage.txt
+examples/test
\ No newline at end of file
diff --git a/.golangci.yml b/.golangci.yml
new file mode 100644
index 0000000..07d6b09
--- /dev/null
+++ b/.golangci.yml
@@ -0,0 +1,172 @@
+service:
+ # When updating this, also update bin/linters.sh accordingly
+ golangci-lint-version: 1.16.x # use the fixed version to not introduce new linters unexpectedly
+run:
+ # timeout for analysis, e.g. 30s, 5m, default is 1m
+ deadline: 20m
+
+ # which dirs to skip: they won't be analyzed;
+ # can use regexp here: generated.*, regexp is applied on full path;
+ # default value is empty list, but next dirs are always skipped independently
+ # from this option's value:
+ # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+
+linters:
+ enable-all: true
+ disable:
+ - depguard
+ - dupl
+ - gochecknoglobals
+ - gochecknoinits
+ - goconst
+ - gocyclo
+ - gosec
+ - nakedret
+ - prealloc
+ - scopelint
+ fast: false
+
+linters-settings:
+ errcheck:
+ # report about not checking of errors in type assetions: `a := b.(MyStruct)`;
+ # default is false: such cases aren't reported by default.
+ check-type-assertions: false
+
+ # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`;
+ # default is false: such cases aren't reported by default.
+ check-blank: false
+ govet:
+ # report about shadowed variables
+ check-shadowing: false
+ golint:
+ # minimal confidence for issues, default is 0.8
+ min-confidence: 0.0
+ gofmt:
+ # simplify code: gofmt with `-s` option, true by default
+ simplify: true
+ goimports:
+ # put imports beginning with prefix after 3rd-party packages;
+ # it's a comma-separated list of prefixes
+ local-prefixes: istio.io/
+ maligned:
+ # print struct with more effective memory layout or not, false by default
+ suggest-new: true
+ misspell:
+ # Correct spellings using locale preferences for US or UK.
+ # Default is to use a neutral variety of English.
+ # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+ locale: US
+ lll:
+ # max line length, lines longer will be reported. Default is 120.
+ # '\t' is counted as 1 character by default, and can be changed with the tab-width option
+ line-length: 160
+ # tab width in spaces. Default to 1.
+ tab-width: 1
+ unused:
+ # treat code as a program (not a library) and report unused exported identifiers; default is false.
+ # XXX: if you enable this setting, unused will report a lot of false-positives in text editors:
+ # if it's called for subdir of a project it can't find funcs usages. All text editor integrations
+ # with golangci-lint call it on a directory with the changed file.
+ check-exported: false
+ unparam:
+ # call graph construction algorithm (cha, rta). In general, use cha for libraries,
+ # and rta for programs with main packages. Default is cha.
+ algo: cha
+
+ # Inspect exported functions, default is false. Set to true if no external program/library imports your code.
+ # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors:
+ # if it's called for subdir of a project it can't find external interfaces. All text editor integrations
+ # with golangci-lint call it on a directory with the changed file.
+ check-exported: false
+ gocritic:
+ enabled-checks:
+ - appendCombine
+ - argOrder
+ - assignOp
+ - badCond
+ - boolExprSimplify
+ - builtinShadow
+ - captLocal
+ - caseOrder
+ - codegenComment
+ - commentedOutCode
+ - commentedOutImport
+ - defaultCaseOrder
+ - deprecatedComment
+ - docStub
+ - dupArg
+ - dupBranchBody
+ - dupCase
+ - dupSubExpr
+ - elseif
+ - emptyFallthrough
+ - equalFold
+ - flagDeref
+ - flagName
+ - hexLiteral
+ - indexAlloc
+ - initClause
+ - methodExprCall
+ - nilValReturn
+ - octalLiteral
+ - offBy1
+ - rangeExprCopy
+ - regexpMust
+ - sloppyLen
+ - stringXbytes
+ - switchTrue
+ - typeAssertChain
+ - typeSwitchVar
+ - typeUnparen
+ - underef
+ - unlambda
+ - unnecessaryBlock
+ - unslice
+ - valSwap
+ - weakCond
+ - yodaStyleExpr
+
+ # Unused
+ # - appendAssign
+ # - commentFormatting
+ # - emptyStringTest
+ # - exitAfterDefer
+ # - ifElseChain
+ # - hugeParam
+ # - importShadow
+ # - nestingReduce
+ # - paramTypeCombine
+ # - ptrToRefParam
+ # - rangeValCopy
+ # - singleCaseSwitch
+ # - sloppyReassign
+ # - unlabelStmt
+ # - unnamedResult
+ # - wrapperFunc
+
+issues:
+ # List of regexps of issue texts to exclude, empty list by default.
+ # But independently from this option we use default exclude patterns,
+ # it can be disabled by `exclude-use-default: false`. To list all
+ # excluded by default patterns execute `golangci-lint run --help`
+ exclude:
+ - composite literal uses unkeyed fields
+
+ exclude-rules:
+ # Exclude some linters from running on test files.
+ - path: _test\.go$|^tests/|^samples/
+ linters:
+ - errcheck
+ - maligned
+
+ # Independently from option `exclude` we use default exclude patterns,
+ # it can be disabled by this option. To list all
+ # excluded by default patterns execute `golangci-lint run --help`.
+ # Default value for this option is true.
+ exclude-use-default: true
+
+ # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+ max-per-linter: 0
+
+ # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+ max-same-issues: 0
diff --git a/.travis.yml b/.travis.yml
index 839052e..d4ed1ac 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,46 +1,37 @@
language: go
-dist: trusty
go:
- - "1.10.x"
- "1.11.x"
-
-go_import_path: github.com/apache/rocketmq-client-go
-
+ - "1.12.x"
+ - "1.13.x"
+go_import_path: github.com/apache/rocketmq-client-go/v2
env:
global:
- NAME_SERVER_ADDRESS=127.0.0.1:9876
- BROKER_ADDRESS=127.0.0.1:10911
- TOPIC=test
- GROUP=testGroup
+ - GO111MODULE=on
+ - HOME=$TRAVIS_HOME
matrix:
- - OS_TYPE=centos OS_VERSION=7
-
-before_install:
- - cd ${TRAVIS_HOME}
- - wget https://opensource-rocketmq-client-us.oss-us-west-1.aliyuncs.com/cpp-client/linux/1.2.4/RHEL7.x/librocketmq.tar.gz
- - tar -xzf librocketmq.tar.gz
- - sudo cp librocketmq.so librocketmq.a /usr/local/lib/
- - sudo cp -r rocketmq /usr/local/include/
- - cd ${GOPATH}/src/github.com/apache/rocketmq-client-go
+ - OS_TYPE=centos OS_VERSION=7
before_script:
- cd ${TRAVIS_HOME}
- - wget https://archive.apache.org/dist/rocketmq/4.3.2/rocketmq-all-4.3.2-bin-release.zip
- - unzip rocketmq-all-4.3.2-bin-release.zip
- - cd rocketmq-all-4.3.2-bin-release
+ - wget https://archive.apache.org/dist/rocketmq/4.6.0/rocketmq-all-4.6.0-bin-release.zip
+ - unzip rocketmq-all-4.6.0-bin-release.zip
+ - cd rocketmq-all-4.6.0-bin-release
- perl -i -pe's/-Xms8g -Xmx8g -Xmn4g/-Xms2g -Xmx2g -Xmn1g/g' bin/runbroker.sh
- nohup sh bin/mqnamesrv &
- - nohup sh bin/mqbroker -n 127.0.0.1:9876 &
- - sleep 10
- - sh bin/mqadmin updateTopic -b '127.0.0.1:10911' –n '127.0.0.1:9876' -t test
- - sh bin/mqadmin updateSubGroup -b '127.0.0.1:10911' –n '127.0.0.1:9876' -g testGroup
- - cd ..
+ - nohup sh bin/mqbroker -n localhost:9876 &
script:
- - export LD_LIBRARY_PATH=/usr/local/lib
- - cd ${GOPATH}/src/github.com/apache/rocketmq-client-go
- - go test ./core -coverprofile=coverage.txt -covermode=atomic
+ - cd ${TRAVIS_HOME}
+ - ls -al
+ - cd ${GOPATH}/src/github.com/apache/rocketmq-client-go/v2
+ - ls -al
+ - go fmt ./... && [[ -z `git status -s` ]]
+ - go mod vendor && go test ./... -coverprofile=coverage.txt -covermode=atomic
after_success:
- - bash <(curl -s https://codecov.io/bash)
\ No newline at end of file
+ - bash <(curl -s https://codecov.io/bash)
diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000..7f27427
--- /dev/null
+++ b/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,19 @@
+## What is the purpose of the change
+
+XXXXX
+
+## Brief changelog
+
+XX
+
+## Verifying this change
+
+XXXX
+
+Follow this checklist to help us incorporate your contribution quickly and easily. Notice, `it would be helpful if you could finish the following 5 checklist(the last one is not necessary)before request the community to review your PR`.
+
+- [x] Make sure there is a [Github issue](https://github.com/apache/rocketmq/issues) filed for the change (usually before you start working on it). Trivial changes like typos do not require a Github issue. Your pull request should address just this issue, without pulling in other changes - one PR resolves one issue.
+- [x] Format the pull request title like `[ISSUE #123] Fix UnknownException when host config not exist`. Each commit in the pull request should have a meaningful subject line and body.
+- [x] Write a pull request description that is detailed enough to understand what the pull request does, how, and why.
+- [x] Write necessary unit-test(over 80% coverage) to verify your logic correction, more mock a little better when a cross-module dependency exists.
+- [ ] If this contribution is large, please file an [Apache Individual Contributor License Agreement](http://www.apache.org/licenses/#clas).
diff --git a/README.md b/README.md
index 98f01c4..4ff24c9 100644
--- a/README.md
+++ b/README.md
@@ -1,35 +1,32 @@
-## RocketMQ Client Go
-[](https://travis-ci.org/apache/rocketmq-client-go)
+## RocketMQ Client Go
[](https://www.apache.org/licenses/LICENSE-2.0.html)
-[](https://goreportcard.com/report/github.com/apache/rocketmq-client-go)
-[](https://godoc.org/github.com/apache/rocketmq-client-go)
-[](https://codecov.io/gh/apache/rocketmq-client-go)
+[](https://travis-ci.org/apache/rocketmq-client-go)
+[](https://codecov.io/gh/apache/rocketmq-client-go/branch/native)
[](https://github.com/apache/rocketmq-client-go/releases)
[](http://isitmaintained.com/project/apache/rocketmq-client-go "Average time to resolve an issue")
[](http://isitmaintained.com/project/apache/rocketmq-client-go "Percentage of issues still open")
-(The Apache RocketMQ Client in Pure Go has been released, Welcome have a try on [the native version](https://github.com/apache/rocketmq-client-go/tree/native))
+This is the first product ready RocketMQ Client in pure go, it supports almost the full features of Apache RocketMQ, sunch as pub and sub messages, ACL, tracing and so on, there are many works need to continue for this project, like unit test, integration test, stable test, new feature,
+optimization, documents, etc. and any contribution is very welcome. if you want do something, please browse issue list and select one, or create a new issue.
-* The client is using cgo to call [rocketmq-client-cpp](https://github.com/apache/rocketmq-client-cpp), which has been proven robust and widely adopted within Alibaba Group by many business units for more than three years.
-----------
-## [Due Diligence](https://github.com/apache/rocketmq-client-go/issues/423)
[Here](https://github.com/apache/rocketmq-client-go/issues/423), we sincerely invite you to take a minute to feedback on your usage scenario.
[Click Here](https://github.com/apache/rocketmq-client-go/issues/423) or go to [ISSUE #423](https://github.com/apache/rocketmq-client-go/issues/423) if you accept.
----------
## Features
-At present, this SDK supports
+For 2.0.0 version, it supports:
* sending message in synchronous mode
-* sending message in orderly mode
+* sending message in asynchronous mode
* sending message in oneway mode
-* sending transaction message
+* sending orderly messages
* consuming message using push model
-* consuming message using pull model(depends cpp core)
+* message tracing for pub and sub messages
+* ACL for producers and consumers
----------
## How to use
-* Step-by-step instruction are provided in [RocketMQ Go Client Introduction](./doc/Introduction.md)
+* Step-by-step instruction are provided in [RocketMQ Go Client Introduction](docs/Introduction.md)
* Consult [RocketMQ Quick Start](https://rocketmq.apache.org/docs/quick-start/) to setup rocketmq broker and nameserver.
----------
@@ -41,7 +38,7 @@
* Mailing Lists: <https://rocketmq.apache.org/about/contact/>
* Home: <https://rocketmq.apache.org>
* Docs: <https://rocketmq.apache.org/docs/quick-start/>
-* Issues: <https://github.com/apache/rocketmq-client-go/issues>
+* Issues: <https://github.com/apache/rocketmq-client-go/v2/issues>
* Ask: <https://stackoverflow.com/questions/tagged/rocketmq>
* Slack: <https://rocketmq-community.slack.com/>
diff --git a/api.go b/api.go
new file mode 100644
index 0000000..0e149e9
--- /dev/null
+++ b/api.go
@@ -0,0 +1,138 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rocketmq
+
+import (
+ "context"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+type Producer interface {
+ Start() error
+ Shutdown() error
+ SendSync(ctx context.Context, mq ...*primitive.Message) (*primitive.SendResult, error)
+ SendAsync(ctx context.Context, mq func(ctx context.Context, result *primitive.SendResult, err error),
+ msg ...*primitive.Message) error
+ SendOneWay(ctx context.Context, mq ...*primitive.Message) error
+}
+
+func NewProducer(opts ...producer.Option) (Producer, error) {
+ return producer.NewDefaultProducer(opts...)
+}
+
+type TransactionProducer interface {
+ Start() error
+ Shutdown() error
+ SendMessageInTransaction(ctx context.Context, mq *primitive.Message) (*primitive.TransactionSendResult, error)
+}
+
+func NewTransactionProducer(listener primitive.TransactionListener, opts ...producer.Option) (TransactionProducer, error) {
+ return producer.NewTransactionProducer(listener, opts...)
+}
+
+type PushConsumer interface {
+ // Start the PullConsumer for consuming message
+ Start() error
+
+ // Shutdown the PullConsumer, all offset of MessageQueue will be sync to broker before process exit
+ Shutdown() error
+ // Subscribe a topic for consuming
+ Subscribe(topic string, selector consumer.MessageSelector,
+ f func(context.Context, ...*primitive.MessageExt) (consumer.ConsumeResult, error)) error
+
+ // Unsubscribe a topic
+ Unsubscribe(topic string) error
+}
+
+func NewPushConsumer(opts ...consumer.Option) (PushConsumer, error) {
+ return consumer.NewPushConsumer(opts...)
+}
+
+type PullConsumer interface {
+ // Start the PullConsumer for consuming message
+ Start() error
+
+ // Shutdown the PullConsumer, all offset of MessageQueue will be commit to broker before process exit
+ Shutdown() error
+
+ // Subscribe a topic for consuming
+ Subscribe(topic string, selector consumer.MessageSelector) error
+
+ // Unsubscribe a topic
+ Unsubscribe(topic string) error
+
+ // MessageQueues get MessageQueue list about for a given topic. This method will issue a remote call to the server
+ // if it does not already have any MessageQueue about the given topic.
+ MessageQueues(topic string) []primitive.MessageQueue
+
+ // Pull message for the topic specified. It is an error to not have subscribed to any topics before pull for message
+ //
+ // Specified numbers of messages is returned if message greater that numbers, and the offset will auto forward.
+ // It means that if you meeting messages consuming failed, you should process failed messages by yourself.
+ Pull(ctx context.Context, topic string, numbers int) (*primitive.PullResult, error)
+
+ // Pull message for the topic specified from a specified MessageQueue and offset. It is an error to not have
+ // subscribed to any topics before pull for message. the method will not affect the offset recorded
+ //
+ // Specified numbers of messages is returned.
+ PullFrom(ctx context.Context, mq primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
+
+ // Lookup offset for the given message queue by timestamp. The returned offset for the message queue is the
+ // earliest offset whose timestamp is greater than or equal to the given timestamp in the corresponding message
+ // queue.
+ //
+ // Timestamp must be millisecond level, if you want to lookup the earliest offset of the mq, you could set the
+ // timestamp 0, and if you want to the latest offset the mq, you could set the timestamp math.MaxInt64.
+ Lookup(ctx context.Context, mq primitive.MessageQueue, timestamp int64) (int64, error)
+
+ // Commit the offset of specified mqs to broker, if auto-commit is disable, you must commit the offset manually.
+ Commit(ctx context.Context, mqs ...primitive.MessageQueue) (int64, error)
+
+ // CommittedOffset return the offset of specified Message
+ CommittedOffset(mq primitive.MessageQueue) (int64, error)
+
+ // Seek set offset of the mq, if you wanna re-consuming your message form one position, the method may help you.
+ // if you want re-consuming from one time, you cloud Lookup() then seek it.
+ Seek(mq primitive.MessageQueue, offset int64) error
+
+ // Pause consuming for specified MessageQueues, after pause, client will not fetch any message from the specified
+ // message queues
+ //
+ // Note that this method does not affect message queue subscription. In particular, it does not cause a group
+ // rebalance.
+ //
+ // if a MessageQueue belong a topic that has not been subscribed, an error will be returned
+ //Pause(mqs ...primitive.MessageQueue) error
+
+ // Resume specified message queues which have been paused with Pause, if a MessageQueue that not paused,
+ // it will be ignored. if not subscribed, an error will be returned
+ //Resume(mqs ...primitive.MessageQueue) error
+}
+
+// The PullConsumer has not implemented completely, if you want have an experience of PullConsumer, you could use
+// consumer.NewPullConsumer(...), but it may changed in the future.
+//
+// The PullConsumer will be supported in next release
+func NewPullConsumer(opts ...consumer.Option) (PullConsumer, error) {
+ return nil, errors.New("pull consumer has not supported")
+}
diff --git a/benchmark/consumer.go b/benchmark/consumer.go
index 73da836..941713c 100644
--- a/benchmark/consumer.go
+++ b/benchmark/consumer.go
@@ -26,8 +26,6 @@
"sync/atomic"
"syscall"
"time"
-
- rocketmq "github.com/apache/rocketmq-client-go/core"
)
type statiBenchmarkConsumerSnapshot struct {
@@ -126,49 +124,49 @@
}
func (c *consumer) consumeMsg(stati *statiBenchmarkConsumerSnapshot, exit chan struct{}) {
- consumer, err := rocketmq.NewPushConsumer(&rocketmq.PushConsumerConfig{
- ClientConfig: rocketmq.ClientConfig{
- GroupID: c.groupID,
- NameServer: c.nameSrv,
- },
- ThreadCount: c.instanceCount,
- MessageBatchMaxSize: 16,
- })
- if err != nil {
- panic("new push consumer error:" + err.Error())
- }
-
- consumer.Subscribe(c.topic, c.expression, func(m *rocketmq.MessageExt) rocketmq.ConsumeStatus {
- atomic.AddInt64(&stati.receiveMessageTotal, 1)
- now := time.Now().UnixNano() / int64(time.Millisecond)
- b2cRT := now - m.BornTimestamp
- atomic.AddInt64(&stati.born2ConsumerTotalRT, b2cRT)
- s2cRT := now - m.StoreTimestamp
- atomic.AddInt64(&stati.store2ConsumerTotalRT, s2cRT)
-
- for {
- old := atomic.LoadInt64(&stati.born2ConsumerMaxRT)
- if old >= b2cRT || atomic.CompareAndSwapInt64(&stati.born2ConsumerMaxRT, old, b2cRT) {
- break
- }
- }
-
- for {
- old := atomic.LoadInt64(&stati.store2ConsumerMaxRT)
- if old >= s2cRT || atomic.CompareAndSwapInt64(&stati.store2ConsumerMaxRT, old, s2cRT) {
- break
- }
- }
-
- return rocketmq.ConsumeSuccess
- })
- println("Start")
- consumer.Start()
- select {
- case <-exit:
- consumer.Shutdown()
- return
- }
+ //consumer, err := rocketmq.NewPushConsumer(&rocketmq.PushConsumerConfig{
+ // ClientConfig: rocketmq.ClientConfig{
+ // GroupID: c.groupID,
+ // NameServer: c.nameSrv,
+ // },
+ // ThreadCount: c.instanceCount,
+ // MessageBatchMaxSize: 16,
+ //})
+ //if err != nil {
+ // panic("new push consumer error:" + err.Error())
+ //}
+ //
+ //consumer.Subscribe(c.topic, c.expression, func(m *rocketmq.MessageExt) rocketmq.ConsumeStatus {
+ // atomic.AddInt64(&stati.receiveMessageTotal, 1)
+ // now := time.Now().UnixNano() / int64(time.Millisecond)
+ // b2cRT := now - m.BornTimestamp
+ // atomic.AddInt64(&stati.born2ConsumerTotalRT, b2cRT)
+ // s2cRT := now - m.StoreTimestamp
+ // atomic.AddInt64(&stati.store2ConsumerTotalRT, s2cRT)
+ //
+ // for {
+ // old := atomic.LoadInt64(&stati.born2ConsumerMaxRT)
+ // if old >= b2cRT || atomic.CompareAndSwapInt64(&stati.born2ConsumerMaxRT, old, b2cRT) {
+ // break
+ // }
+ // }
+ //
+ // for {
+ // old := atomic.LoadInt64(&stati.store2ConsumerMaxRT)
+ // if old >= s2cRT || atomic.CompareAndSwapInt64(&stati.store2ConsumerMaxRT, old, s2cRT) {
+ // break
+ // }
+ // }
+ //
+ // return rocketmq.ConsumeSuccess
+ //})
+ //println("Start")
+ //consumer.Start()
+ //select {
+ //case <-exit:
+ // consumer.Shutdown()
+ // return
+ //}
}
func (c *consumer) run(args []string) {
@@ -214,15 +212,15 @@
wg := sync.WaitGroup{}
+ wg.Add(1)
go func() {
- wg.Add(1)
c.consumeMsg(&stati, exitChan)
wg.Done()
}()
// snapshot
+ wg.Add(1)
go func() {
- wg.Add(1)
defer wg.Done()
ticker := time.NewTicker(time.Second)
for {
@@ -237,8 +235,8 @@
}()
// print statistic
+ wg.Add(1)
go func() {
- wg.Add(1)
defer wg.Done()
ticker := time.NewTicker(time.Second * 10)
for {
diff --git a/benchmark/producer.go b/benchmark/producer.go
index e183269..f3c1c60 100644
--- a/benchmark/producer.go
+++ b/benchmark/producer.go
@@ -26,8 +26,6 @@
"sync/atomic"
"syscall"
"time"
-
- rocketmq "github.com/apache/rocketmq-client-go/core"
)
type statiBenchmarkProducerSnapshot struct {
@@ -108,7 +106,7 @@
func init() {
p := &producer{}
- flags := flag.NewFlagSet("producer", flag.ExitOnError)
+ flags := flag.NewFlagSet("consumer", flag.ExitOnError)
p.flags = flags
flags.StringVar(&p.topic, "t", "", "topic name")
@@ -118,57 +116,57 @@
flags.IntVar(&p.testMinutes, "m", 10, "test minutes")
flags.IntVar(&p.bodySize, "s", 32, "body size")
- registerCommand("producer", p)
+ registerCommand("consumer", p)
}
func (bp *producer) produceMsg(stati *statiBenchmarkProducerSnapshot, exit chan struct{}) {
- p, err := rocketmq.NewProducer(&rocketmq.ProducerConfig{
- ClientConfig: rocketmq.ClientConfig{GroupID: bp.groupID, NameServer: bp.nameSrv},
- })
- if err != nil {
- fmt.Printf("new producer error:%s\n", err)
- return
- }
+ //p, err := rocketmq.NewProducer(&rocketmq.ProducerConfig{
+ // ClientConfig: rocketmq.ClientConfig{GroupID: bp.groupID, NameServer: bp.nameSrv},
+ //})
+ //if err != nil {
+ // fmt.Printf("new consumer error:%s\n", err)
+ // return
+ //}
+ //
+ //p.Start()
+ //defer p.Shutdown()
- p.Start()
- defer p.Shutdown()
+ //topic, tag := bp.topic, "benchmark-consumer"
+ //
+ //AGAIN:
+ // select {
+ // case <-exit:
+ // return
+ // default:
+ // }
- topic, tag := bp.topic, "benchmark-producer"
-
-AGAIN:
- select {
- case <-exit:
- return
- default:
- }
-
- now := time.Now()
- r, err := p.SendMessageSync(&rocketmq.Message{
- Topic: bp.topic, Body: buildMsg(bp.bodySize),
- })
-
- if err != nil {
- fmt.Printf("send message sync error:%s", err)
- goto AGAIN
- }
-
- if r.Status == rocketmq.SendOK {
- atomic.AddInt64(&stati.receiveResponseSuccessCount, 1)
- atomic.AddInt64(&stati.sendRequestSuccessCount, 1)
- currentRT := int64(time.Since(now) / time.Millisecond)
- atomic.AddInt64(&stati.sendMessageSuccessTimeTotal, currentRT)
- prevRT := atomic.LoadInt64(&stati.sendMessageMaxRT)
- for currentRT > prevRT {
- if atomic.CompareAndSwapInt64(&stati.sendMessageMaxRT, prevRT, currentRT) {
- break
- }
- prevRT = atomic.LoadInt64(&stati.sendMessageMaxRT)
- }
- goto AGAIN
- }
-
- fmt.Printf("%v send message %s:%s error:%s\n", time.Now(), topic, tag, err.Error())
- goto AGAIN
+ //now := time.Now()
+ //r, err := p.SendMessageSync(&rocketmq.Message{
+ // Topic: bp.topic, Body: buildMsg(bp.bodySize),
+ //})
+ //
+ //if err != nil {
+ // fmt.Printf("send message sync error:%s", err)
+ // goto AGAIN
+ //}
+ //
+ //if r.Status == rocketmq.SendOK {
+ // atomic.AddInt64(&stati.receiveResponseSuccessCount, 1)
+ // atomic.AddInt64(&stati.sendRequestSuccessCount, 1)
+ // currentRT := int64(time.Since(now) / time.Millisecond)
+ // atomic.AddInt64(&stati.sendMessageSuccessTimeTotal, currentRT)
+ // prevRT := atomic.LoadInt64(&stati.sendMessageMaxRT)
+ // for currentRT > prevRT {
+ // if atomic.CompareAndSwapInt64(&stati.sendMessageMaxRT, prevRT, currentRT) {
+ // break
+ // }
+ // prevRT = atomic.LoadInt64(&stati.sendMessageMaxRT)
+ // }
+ // goto AGAIN
+ //}
+ //
+ //fmt.Printf("%v send message %s:%s error:%s\n", time.Now(), topic, tag, err.Error())
+ //goto AGAIN
}
func (bp *producer) run(args []string) {
@@ -223,8 +221,8 @@
}
// snapshot
+ wg.Add(1)
go func() {
- wg.Add(1)
defer wg.Done()
ticker := time.NewTicker(time.Second)
for {
@@ -239,8 +237,8 @@
}()
// print statistic
+ wg.Add(1)
go func() {
- wg.Add(1)
defer wg.Done()
ticker := time.NewTicker(time.Second * 10)
for {
diff --git a/benchmark/stable.go b/benchmark/stable.go
index 6c7a12c..2c3dc1b 100644
--- a/benchmark/stable.go
+++ b/benchmark/stable.go
@@ -25,8 +25,6 @@
"os/signal"
"syscall"
"time"
-
- rocketmq "github.com/apache/rocketmq-client-go/core"
)
type stableTest struct {
@@ -102,7 +100,7 @@
*stableTest
bodySize int
- p rocketmq.Producer
+ //p rocketmq.Producer
}
func (stp *stableTestProducer) buildFlags(name string) {
@@ -141,39 +139,39 @@
return
}
- p, err := rocketmq.NewProducer(&rocketmq.ProducerConfig{
- ClientConfig: rocketmq.ClientConfig{GroupID: stp.groupID, NameServer: stp.nameSrv},
- })
- if err != nil {
- fmt.Printf("new producer error:%s\n", err)
- return
- }
-
- err = p.Start()
- if err != nil {
- fmt.Printf("start producer error:%s\n", err)
- return
- }
- defer p.Shutdown()
-
- stp.p = p
+ //p, err := rocketmq.NewProducer(&rocketmq.ProducerConfig{
+ // ClientConfig: rocketmq.ClientConfig{GroupID: stp.groupID, NameServer: stp.nameSrv},
+ //})
+ //if err != nil {
+ // fmt.Printf("new consumer error:%s\n", err)
+ // return
+ //}
+ //
+ //err = p.Start()
+ //if err != nil {
+ // fmt.Printf("start consumer error:%s\n", err)
+ // return
+ //}
+ //defer p.Shutdown()
+ //
+ //stp.p = p
stp.stableTest.run()
}
func (stp *stableTestProducer) sendMessage() {
- r, err := stp.p.SendMessageSync(&rocketmq.Message{Topic: stp.topic, Body: buildMsg(stp.bodySize)})
- if err == nil {
- fmt.Printf("send result:%+v\n", r)
- return
- }
- fmt.Printf("send message error:%s", err)
+ //r, err := stp.p.SendMessageSync(&rocketmq.Message{Topic: stp.topic, Body: buildMsg(stp.bodySize)})
+ //if err == nil {
+ // fmt.Printf("send result:%+v\n", r)
+ // return
+ //}
+ //fmt.Printf("send message error:%s", err)
}
type stableTestConsumer struct {
*stableTest
expression string
- c rocketmq.PullConsumer
+ //c rocketmq.PullConsumer
offsets map[int]int64
}
@@ -212,51 +210,51 @@
fmt.Printf("%s\n", err)
return
}
-
- c, err := rocketmq.NewPullConsumer(&rocketmq.PullConsumerConfig{
- ClientConfig: rocketmq.ClientConfig{GroupID: stc.groupID, NameServer: stc.nameSrv},
- })
- if err != nil {
- fmt.Printf("new pull consumer error:%s\n", err)
- return
- }
-
- err = c.Start()
- if err != nil {
- fmt.Printf("start consumer error:%s\n", err)
- return
- }
- defer c.Shutdown()
-
- stc.c = c
+ //
+ //c, err := rocketmq.NewPullConsumer(&rocketmq.PullConsumerConfig{
+ // ClientConfig: rocketmq.ClientConfig{GroupID: stc.groupID, NameServer: stc.nameSrv},
+ //})
+ //if err != nil {
+ // fmt.Printf("new pull consumer error:%s\n", err)
+ // return
+ //}
+ //
+ //err = c.Start()
+ //if err != nil {
+ // fmt.Printf("start consumer error:%s\n", err)
+ // return
+ //}
+ //defer c.Shutdown()
+ //
+ //stc.c = c
stc.stableTest.run()
}
func (stc *stableTestConsumer) pullMessage() {
- mqs := stc.c.FetchSubscriptionMessageQueues(stc.topic)
-
- for _, mq := range mqs {
- offset := stc.offsets[mq.ID]
- pr := stc.c.Pull(mq, stc.expression, offset, 32)
- fmt.Printf("pull from %s, offset:%d, count:%+v\n", mq.String(), offset, len(pr.Messages))
-
- switch pr.Status {
- case rocketmq.PullNoNewMsg:
- stc.offsets[mq.ID] = 0 // pull from the begin
- case rocketmq.PullFound:
- fallthrough
- case rocketmq.PullNoMatchedMsg:
- fallthrough
- case rocketmq.PullOffsetIllegal:
- stc.offsets[mq.ID] = pr.NextBeginOffset
- case rocketmq.PullBrokerTimeout:
- fmt.Println("broker timeout occur")
- }
- }
+ //mqs := stc.c.FetchSubscriptionMessageQueues(stc.topic)
+ //
+ //for _, mq := range mqs {
+ // offset := stc.offsets[mq.ID]
+ // pr := stc.c.Pull(mq, stc.expression, offset, 32)
+ //fmt.Printf("pull from %s, offset:%d, count:%+v\n", mq.String(), offset, len(pr.Messages))
+ //
+ //switch pr.Status {
+ //case rocketmq.PullNoNewMsg:
+ // stc.offsets[mq.ID] = 0 // pull from the begin
+ //case rocketmq.PullFound:
+ // fallthrough
+ //case rocketmq.PullNoMatchedMsg:
+ // fallthrough
+ //case rocketmq.PullOffsetIllegal:
+ // stc.offsets[mq.ID] = pr.NextBeginOffset
+ //case rocketmq.PullBrokerTimeout:
+ // fmt.Println("broker timeout occur")
+ //}
+ //}
}
func init() {
- // producer
+ // consumer
name := "stableTestProducer"
p := &stableTestProducer{stableTest: &stableTest{}}
p.buildFlags(name)
diff --git a/config.go b/config.go
new file mode 100644
index 0000000..8f1592c
--- /dev/null
+++ b/config.go
@@ -0,0 +1,17 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package rocketmq
diff --git a/consumer/consumer.go b/consumer/consumer.go
new file mode 100644
index 0000000..3129b8d
--- /dev/null
+++ b/consumer/consumer.go
@@ -0,0 +1,1061 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "context"
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ jsoniter "github.com/json-iterator/go"
+
+ "github.com/pkg/errors"
+ "github.com/tidwall/gjson"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+const (
+ // Delay some time when exception error
+ _PullDelayTimeWhenError = 3 * time.Second
+
+ // Flow control interval
+ _PullDelayTimeWhenFlowControl = 50 * time.Millisecond
+
+ // Delay some time when suspend pull service
+ _PullDelayTimeWhenSuspend = 30 * time.Second
+
+ // Long polling mode, the Consumer connection max suspend time
+ _BrokerSuspendMaxTime = 20 * time.Second
+
+ // Long polling mode, the Consumer connection timeout (must greater than _BrokerSuspendMaxTime)
+ _ConsumerTimeoutWhenSuspend = 30 * time.Second
+
+ // Offset persistent interval for consumer
+ _PersistConsumerOffsetInterval = 5 * time.Second
+)
+
+type ConsumeType string
+
+const (
+ _PullConsume = ConsumeType("CONSUME_ACTIVELY")
+ _PushConsume = ConsumeType("CONSUME_PASSIVELY")
+
+ _SubAll = "*"
+)
+
+var (
+ ErrCreated = errors.New("consumer group has been created")
+ ErrBrokerNotFound = errors.New("broker can not found")
+)
+
+// Message model defines the way how messages are delivered to each consumer clients.
+// </p>
+//
+// RocketMQ supports two message models: clustering and broadcasting. If clustering is set, consumer clients with
+// the same {@link #ConsumerGroup} would only consume shards of the messages subscribed, which achieves load
+// balances; Conversely, if the broadcasting is set, each consumer client will consume all subscribed messages
+// separately.
+// </p>
+//
+// This field defaults to clustering.
+type MessageModel int
+
+const (
+ BroadCasting MessageModel = iota
+ Clustering
+)
+
+func (mode MessageModel) String() string {
+ switch mode {
+ case BroadCasting:
+ return "BroadCasting"
+ case Clustering:
+ return "Clustering"
+ default:
+ return "Unknown"
+ }
+}
+
+// Consuming point on consumer booting.
+// </p>
+//
+// There are three consuming points:
+// <ul>
+// <li>
+// <code>CONSUME_FROM_LAST_OFFSET</code>: consumer clients pick up where it stopped previously.
+// If it were a newly booting up consumer client, according aging of the consumer group, there are two
+// cases:
+// <ol>
+// <li>
+// if the consumer group is created so recently that the earliest message being subscribed has yet
+// expired, which means the consumer group represents a lately launched business, consuming will
+// start from the very beginning;
+// </li>
+// <li>
+// if the earliest message being subscribed has expired, consuming will start from the latest
+// messages, meaning messages born prior to the booting timestamp would be ignored.
+// </li>
+// </ol>
+// </li>
+// <li>
+// <code>CONSUME_FROM_FIRST_OFFSET</code>: Consumer client will start from earliest messages available.
+// </li>
+// <li>
+// <code>CONSUME_FROM_TIMESTAMP</code>: Consumer client will start from specified timestamp, which means
+// messages born prior to {@link #consumeTimestamp} will be ignored
+// </li>
+// </ul>
+type ConsumeFromWhere int
+
+const (
+ ConsumeFromLastOffset ConsumeFromWhere = iota
+ ConsumeFromFirstOffset
+ ConsumeFromTimestamp
+)
+
+type ExpressionType string
+
+const (
+ /**
+ * <ul>
+ * Keywords:
+ * <li>{@code AND, OR, NOT, BETWEEN, IN, TRUE, FALSE, IS, NULL}</li>
+ * </ul>
+ * <p/>
+ * <ul>
+ * Data type:
+ * <li>Boolean, like: TRUE, FALSE</li>
+ * <li>String, like: 'abc'</li>
+ * <li>Decimal, like: 123</li>
+ * <li>Float number, like: 3.1415</li>
+ * </ul>
+ * <p/>
+ * <ul>
+ * Grammar:
+ * <li>{@code AND, OR}</li>
+ * <li>{@code >, >=, <, <=, =}</li>
+ * <li>{@code BETWEEN A AND B}, equals to {@code >=A AND <=B}</li>
+ * <li>{@code NOT BETWEEN A AND B}, equals to {@code >B OR <A}</li>
+ * <li>{@code IN ('a', 'b')}, equals to {@code ='a' OR ='b'}, this operation only support String type.</li>
+ * <li>{@code IS NULL}, {@code IS NOT NULL}, check parameter whether is null, or not.</li>
+ * <li>{@code =TRUE}, {@code =FALSE}, check parameter whether is true, or false.</li>
+ * </ul>
+ * <p/>
+ * <p>
+ * Example:
+ * (a > 10 AND a < 100) OR (b IS NOT NULL AND b=TRUE)
+ * </p>
+ */
+ SQL92 = ExpressionType("SQL92")
+
+ /**
+ * Only support or operation such as
+ * "tag1 || tag2 || tag3", <br>
+ * If null or * expression, meaning subscribe all.
+ */
+ TAG = ExpressionType("TAG")
+)
+
+func IsTagType(exp string) bool {
+ if exp == "" || exp == "TAG" {
+ return true
+ }
+ return false
+}
+
+type MessageSelector struct {
+ Type ExpressionType
+ Expression string
+}
+
+type ConsumeResult int
+
+const (
+ ConsumeSuccess ConsumeResult = iota
+ ConsumeRetryLater
+ Commit
+ Rollback
+ SuspendCurrentQueueAMoment
+)
+
+type ConsumeResultHolder struct {
+ ConsumeResult
+}
+
+type ConsumerReturn int
+
+const (
+ SuccessReturn ConsumerReturn = iota
+ ExceptionReturn
+ NullReturn
+ TimeoutReturn
+ FailedReturn
+)
+
+type PullRequest struct {
+ consumerGroup string
+ mq *primitive.MessageQueue
+ pq *processQueue
+ nextOffset int64
+ lockedFirst bool
+}
+
+func (pr *PullRequest) String() string {
+ return fmt.Sprintf("[ConsumerGroup: %s, Topic: %s, MessageQueue: %d]",
+ pr.consumerGroup, pr.mq.Topic, pr.mq.QueueId)
+}
+
+type defaultConsumer struct {
+ /**
+ * Consumers of the same role is required to have exactly same subscriptions and consumerGroup to correctly achieve
+ * load balance. It's required and needs to be globally unique.
+ * </p>
+ *
+ * See <a href="http://rocketmq.apache.org/docs/core-concept/">here</a> for further discussion.
+ */
+ consumerGroup string
+ model MessageModel
+ allocate func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
+ unitMode bool
+ consumeOrderly bool
+ fromWhere ConsumeFromWhere
+ consumerStartTimestamp int64
+
+ cType ConsumeType
+ client internal.RMQClient
+ mqChanged func(topic string, mqAll, mqDivided []*primitive.MessageQueue)
+ state int32
+ pause bool
+ once sync.Once
+ option consumerOptions
+ // key: primitive.MessageQueue
+ // value: *processQueue
+ processQueueTable sync.Map
+
+ // key: topic(string)
+ // value: map[int]*primitive.MessageQueue
+ topicSubscribeInfoTable sync.Map
+
+ // key: topic
+ // value: *SubscriptionData
+ subscriptionDataTable sync.Map
+ storage OffsetStore
+ // chan for push consumer
+ prCh chan PullRequest
+
+ namesrv internal.Namesrvs
+
+ pullFromWhichNodeTable sync.Map
+}
+
+func (dc *defaultConsumer) start() error {
+ if dc.model == Clustering {
+ // set retry topic
+ retryTopic := internal.GetRetryTopic(dc.consumerGroup)
+ sub := buildSubscriptionData(retryTopic, MessageSelector{TAG, _SubAll})
+ dc.subscriptionDataTable.Store(retryTopic, sub)
+ }
+
+ if dc.model == Clustering {
+ dc.option.ChangeInstanceNameToPID()
+ dc.storage = NewRemoteOffsetStore(dc.consumerGroup, dc.client, dc.namesrv)
+ } else {
+ dc.storage = NewLocalFileOffsetStore(dc.consumerGroup, dc.client.ClientID())
+ }
+
+ dc.client.Start()
+ atomic.StoreInt32(&dc.state, int32(internal.StateRunning))
+ dc.consumerStartTimestamp = time.Now().UnixNano() / int64(time.Millisecond)
+ return nil
+}
+
+func (dc *defaultConsumer) shutdown() error {
+ atomic.StoreInt32(&dc.state, int32(internal.StateShutdown))
+
+ mqs := make([]*primitive.MessageQueue, 0)
+ dc.processQueueTable.Range(func(key, value interface{}) bool {
+ k := key.(primitive.MessageQueue)
+ pq := value.(*processQueue)
+ pq.WithDropped(true)
+ mqs = append(mqs, &k)
+ return true
+ })
+ dc.storage.persist(mqs)
+ dc.client.Shutdown()
+ return nil
+}
+
+func (dc *defaultConsumer) persistConsumerOffset() error {
+ err := dc.makeSureStateOK()
+ if err != nil {
+ return err
+ }
+ mqs := make([]*primitive.MessageQueue, 0)
+ dc.processQueueTable.Range(func(key, value interface{}) bool {
+ k := key.(primitive.MessageQueue)
+ mqs = append(mqs, &k)
+ return true
+ })
+ dc.storage.persist(mqs)
+ return nil
+}
+
+func (dc *defaultConsumer) updateOffset(queue *primitive.MessageQueue, offset int64) error {
+ dc.storage.update(queue, offset, false)
+ return nil
+}
+
+func (dc *defaultConsumer) subscriptionAutomatically(topic string) {
+ _, exist := dc.subscriptionDataTable.Load(topic)
+ if !exist {
+ s := MessageSelector{
+ Expression: _SubAll,
+ }
+ dc.subscriptionDataTable.Store(topic, buildSubscriptionData(topic, s))
+ }
+}
+
+func (dc *defaultConsumer) updateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
+ _, exist := dc.subscriptionDataTable.Load(topic)
+ if exist {
+ dc.topicSubscribeInfoTable.Store(topic, mqs)
+ }
+}
+
+func (dc *defaultConsumer) isSubscribeTopicNeedUpdate(topic string) bool {
+ _, exist := dc.subscriptionDataTable.Load(topic)
+ if !exist {
+ return false
+ }
+ _, exist = dc.topicSubscribeInfoTable.Load(topic)
+ return !exist
+}
+
+func (dc *defaultConsumer) doBalance() {
+ dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
+ topic := key.(string)
+ v, exist := dc.topicSubscribeInfoTable.Load(topic)
+ if !exist {
+ rlog.Warning("do balance in group failed, the topic does not exist", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyTopic: topic,
+ })
+ return true
+ }
+ mqs := v.([]*primitive.MessageQueue)
+ switch dc.model {
+ case BroadCasting:
+ changed := dc.updateProcessQueueTable(topic, mqs)
+ if changed {
+ dc.mqChanged(topic, mqs, mqs)
+ rlog.Debug("MessageQueue changed", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyTopic: topic,
+ rlog.LogKeyMessageQueue: fmt.Sprintf("%v", mqs),
+ })
+ }
+ case Clustering:
+ cidAll := dc.findConsumerList(topic)
+ if cidAll == nil {
+ rlog.Warning("do balance in group failed, get consumer id list failed", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyTopic: topic,
+ })
+ return true
+ }
+ mqAll := make([]*primitive.MessageQueue, len(mqs))
+ copy(mqAll, mqs)
+ sort.Strings(cidAll)
+ sort.SliceStable(mqAll, func(i, j int) bool {
+ v := strings.Compare(mqAll[i].Topic, mqAll[j].Topic)
+ if v != 0 {
+ return v < 0
+ }
+
+ v = strings.Compare(mqAll[i].BrokerName, mqAll[j].BrokerName)
+ if v != 0 {
+ return v < 0
+ }
+ return (mqAll[i].QueueId - mqAll[j].QueueId) < 0
+ })
+ allocateResult := dc.allocate(dc.consumerGroup, dc.client.ClientID(), mqAll, cidAll)
+ changed := dc.updateProcessQueueTable(topic, allocateResult)
+ if changed {
+ dc.mqChanged(topic, mqAll, allocateResult)
+ rlog.Debug("MessageQueue do balance done", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyTopic: topic,
+ "clientID": dc.client.ClientID(),
+ "mqAllSize": len(mqAll),
+ "cidAllSize": len(cidAll),
+ "rebalanceResultSize": len(allocateResult),
+ "rebalanceResultSet": allocateResult,
+ })
+ }
+ }
+ return true
+ })
+}
+
+func (dc *defaultConsumer) SubscriptionDataList() []*internal.SubscriptionData {
+ result := make([]*internal.SubscriptionData, 0)
+ dc.subscriptionDataTable.Range(func(key, value interface{}) bool {
+ result = append(result, value.(*internal.SubscriptionData))
+ return true
+ })
+ return result
+}
+
+func (dc *defaultConsumer) makeSureStateOK() error {
+ if atomic.LoadInt32(&dc.state) != int32(internal.StateRunning) {
+ return fmt.Errorf("state not running, actually: %v", dc.state)
+ }
+ return nil
+}
+
+type lockBatchRequestBody struct {
+ ConsumerGroup string `json:"consumerGroup"`
+ ClientId string `json:"clientId"`
+ MQs []*primitive.MessageQueue `json:"mqSet"`
+}
+
+func (dc *defaultConsumer) lock(mq *primitive.MessageQueue) bool {
+ brokerResult := dc.namesrv.FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
+
+ if brokerResult == nil {
+ return false
+ }
+
+ body := &lockBatchRequestBody{
+ ConsumerGroup: dc.consumerGroup,
+ ClientId: dc.client.ClientID(),
+ MQs: []*primitive.MessageQueue{mq},
+ }
+ lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
+ var lockOK bool
+ for idx := range lockedMQ {
+ _mq := lockedMQ[idx]
+ v, exist := dc.processQueueTable.Load(_mq)
+ if exist {
+ pq := v.(*processQueue)
+ pq.WithLock(true)
+ pq.UpdateLastConsumeTime()
+ pq.UpdateLastLockTime()
+ }
+ if _mq == *mq {
+ lockOK = true
+ }
+ }
+ fields := map[string]interface{}{
+ "lockOK": lockOK,
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: mq.String(),
+ }
+ if lockOK {
+ rlog.Debug("lock MessageQueue", fields)
+ } else {
+ rlog.Info("lock MessageQueue", fields)
+ }
+ return lockOK
+}
+
+func (dc *defaultConsumer) unlock(mq *primitive.MessageQueue, oneway bool) {
+ brokerResult := dc.namesrv.FindBrokerAddressInSubscribe(mq.BrokerName, internal.MasterId, true)
+
+ if brokerResult == nil {
+ return
+ }
+
+ body := &lockBatchRequestBody{
+ ConsumerGroup: dc.consumerGroup,
+ ClientId: dc.client.ClientID(),
+ MQs: []*primitive.MessageQueue{mq},
+ }
+ dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
+ rlog.Info("unlock MessageQueue", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ "clientID": dc.client.ClientID(),
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+}
+
+func (dc *defaultConsumer) lockAll() {
+ mqMapSet := dc.buildProcessQueueTableByBrokerName()
+ for broker, mqs := range mqMapSet {
+ if len(mqs) == 0 {
+ continue
+ }
+ brokerResult := dc.namesrv.FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
+ if brokerResult == nil {
+ continue
+ }
+ body := &lockBatchRequestBody{
+ ConsumerGroup: dc.consumerGroup,
+ ClientId: dc.client.ClientID(),
+ MQs: mqs,
+ }
+ lockedMQ := dc.doLock(brokerResult.BrokerAddr, body)
+ set := make(map[primitive.MessageQueue]bool)
+ for idx := range lockedMQ {
+ _mq := lockedMQ[idx]
+ v, exist := dc.processQueueTable.Load(_mq)
+ if exist {
+ pq := v.(*processQueue)
+ pq.WithLock(true)
+ pq.UpdateLastConsumeTime()
+ }
+ set[_mq] = true
+ }
+ for idx := range mqs {
+ _mq := mqs[idx]
+ if !set[*_mq] {
+ v, exist := dc.processQueueTable.Load(_mq)
+ if exist {
+ pq := v.(*processQueue)
+ pq.WithLock(false)
+ pq.UpdateLastLockTime()
+ rlog.Info("lock MessageQueue", map[string]interface{}{
+ "lockOK": false,
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: _mq.String(),
+ })
+ }
+ }
+ }
+ }
+}
+
+func (dc *defaultConsumer) unlockAll(oneway bool) {
+ mqMapSet := dc.buildProcessQueueTableByBrokerName()
+ for broker, mqs := range mqMapSet {
+ if len(mqs) == 0 {
+ continue
+ }
+ brokerResult := dc.namesrv.FindBrokerAddressInSubscribe(broker, internal.MasterId, true)
+ if brokerResult == nil {
+ continue
+ }
+ body := &lockBatchRequestBody{
+ ConsumerGroup: dc.consumerGroup,
+ ClientId: dc.client.ClientID(),
+ MQs: mqs,
+ }
+ dc.doUnlock(brokerResult.BrokerAddr, body, oneway)
+ for idx := range mqs {
+ _mq := mqs[idx]
+ v, exist := dc.processQueueTable.Load(_mq)
+ if exist {
+ rlog.Info("lock MessageQueue", map[string]interface{}{
+ "lockOK": false,
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: _mq.String(),
+ })
+ v.(*processQueue).WithLock(false)
+ }
+ }
+ }
+}
+
+func (dc *defaultConsumer) doLock(addr string, body *lockBatchRequestBody) []primitive.MessageQueue {
+ data, _ := jsoniter.Marshal(body)
+ request := remote.NewRemotingCommand(internal.ReqLockBatchMQ, nil, data)
+ response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
+ if err != nil {
+ rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
+ rlog.LogKeyBroker: addr,
+ rlog.LogKeyUnderlayError: err,
+ })
+ return nil
+ }
+ lockOKMQSet := struct {
+ MQs []primitive.MessageQueue `json:"lockOKMQSet"`
+ }{}
+ if len(response.Body) == 0 {
+ return nil
+ }
+ err = jsoniter.Unmarshal(response.Body, &lockOKMQSet)
+ if err != nil {
+ rlog.Error("Unmarshal lock mq body error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ return nil
+ }
+ return lockOKMQSet.MQs
+}
+
+func (dc *defaultConsumer) doUnlock(addr string, body *lockBatchRequestBody, oneway bool) {
+ data, _ := jsoniter.Marshal(body)
+ request := remote.NewRemotingCommand(internal.ReqUnlockBatchMQ, nil, data)
+ if oneway {
+ err := dc.client.InvokeOneWay(context.Background(), addr, request, 3*time.Second)
+ if err != nil {
+ rlog.Error("lock MessageQueue to broker invoke oneway error", map[string]interface{}{
+ rlog.LogKeyBroker: addr,
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ } else {
+ response, err := dc.client.InvokeSync(context.Background(), addr, request, 1*time.Second)
+ rlog.Error("lock MessageQueue to broker invoke error", map[string]interface{}{
+ rlog.LogKeyBroker: addr,
+ rlog.LogKeyUnderlayError: err,
+ })
+ if response.Code != internal.ResSuccess {
+ // TODO error
+ }
+ }
+}
+
+func (dc *defaultConsumer) buildProcessQueueTableByBrokerName() map[string][]*primitive.MessageQueue {
+ result := make(map[string][]*primitive.MessageQueue, 0)
+
+ dc.processQueueTable.Range(func(key, value interface{}) bool {
+ mq := key.(primitive.MessageQueue)
+ mqs, exist := result[mq.BrokerName]
+ if !exist {
+ mqs = make([]*primitive.MessageQueue, 0)
+ }
+ mqs = append(mqs, &mq)
+ result[mq.BrokerName] = mqs
+ return true
+ })
+
+ return result
+}
+
+func (dc *defaultConsumer) updateProcessQueueTable(topic string, mqs []*primitive.MessageQueue) bool {
+ var changed bool
+ mqSet := make(map[primitive.MessageQueue]bool)
+ for idx := range mqs {
+ mqSet[*mqs[idx]] = true
+ }
+ dc.processQueueTable.Range(func(key, value interface{}) bool {
+ mq := key.(primitive.MessageQueue)
+ pq := value.(*processQueue)
+ if mq.Topic == topic {
+ if !mqSet[mq] {
+ pq.WithDropped(true)
+ if dc.removeUnnecessaryMessageQueue(&mq, pq) {
+ dc.processQueueTable.Delete(key)
+ changed = true
+ rlog.Debug("remove unnecessary mq when updateProcessQueueTable", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ }
+ } else if pq.isPullExpired() && dc.cType == _PushConsume {
+ pq.WithDropped(true)
+ if dc.removeUnnecessaryMessageQueue(&mq, pq) {
+ dc.processQueueTable.Delete(key)
+ changed = true
+ rlog.Debug("remove unnecessary mq because pull was paused, prepare to fix it", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ }
+ }
+ }
+ return true
+ })
+
+ if dc.cType == _PushConsume {
+ for item := range mqSet {
+ // BUG: the mq will send to channel, if not copy once, the next iter will modify the mq in the channel.
+ mq := item
+ _, exist := dc.processQueueTable.Load(mq)
+ if exist {
+ continue
+ }
+ if dc.consumeOrderly && !dc.lock(&mq) {
+ rlog.Warning("do defaultConsumer, add a new mq failed, because lock failed", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ continue
+ }
+ dc.storage.remove(&mq)
+ nextOffset := dc.computePullFromWhere(&mq)
+ if nextOffset >= 0 {
+ _, exist := dc.processQueueTable.Load(mq)
+ if exist {
+ rlog.Debug("do defaultConsumer, mq already exist", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ } else {
+ rlog.Debug("do defaultConsumer, add a new mq", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ pq := newProcessQueue(dc.consumeOrderly)
+ dc.processQueueTable.Store(mq, pq)
+ pr := PullRequest{
+ consumerGroup: dc.consumerGroup,
+ mq: &mq,
+ pq: pq,
+ nextOffset: nextOffset,
+ }
+ dc.prCh <- pr
+ changed = true
+ }
+ } else {
+ rlog.Warning("do defaultConsumer, add a new mq failed", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ }
+ }
+ }
+
+ return changed
+}
+
+func (dc *defaultConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
+ dc.storage.persist([]*primitive.MessageQueue{mq})
+ dc.storage.remove(mq)
+ return true
+}
+
+func (dc *defaultConsumer) computePullFromWhere(mq *primitive.MessageQueue) int64 {
+ if dc.cType == _PullConsume {
+ return 0
+ }
+ var result = int64(-1)
+ lastOffset := dc.storage.read(mq, _ReadFromStore)
+ if lastOffset >= 0 {
+ result = lastOffset
+ } else {
+ switch dc.option.FromWhere {
+ case ConsumeFromLastOffset:
+ if lastOffset == -1 {
+ if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
+ lastOffset = 0
+ } else {
+ lastOffset, err := dc.queryMaxOffset(mq)
+ if err == nil {
+ result = lastOffset
+ } else {
+ rlog.Warning("query max offset error", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq,
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ }
+ } else {
+ result = -1
+ }
+ case ConsumeFromFirstOffset:
+ if lastOffset == -1 {
+ result = 0
+ }
+ case ConsumeFromTimestamp:
+ if lastOffset == -1 {
+ if strings.HasPrefix(mq.Topic, internal.RetryGroupTopicPrefix) {
+ lastOffset, err := dc.queryMaxOffset(mq)
+ if err == nil {
+ result = lastOffset
+ } else {
+ result = -1
+ rlog.Warning("query max offset error", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq,
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ } else {
+ t, err := time.Parse("20060102150405", dc.option.ConsumeTimestamp)
+ if err != nil {
+ result = -1
+ } else {
+ lastOffset, err := dc.searchOffsetByTimestamp(mq, t.Unix()*1000)
+ if err != nil {
+ result = -1
+ } else {
+ result = lastOffset
+ }
+ }
+ }
+ }
+ default:
+ }
+ }
+ return result
+}
+
+func (dc *defaultConsumer) pullInner(ctx context.Context, queue *primitive.MessageQueue, data *internal.SubscriptionData,
+ offset int64, numbers int, sysFlag int32, commitOffsetValue int64) (*primitive.PullResult, error) {
+
+ brokerResult := dc.tryFindBroker(queue)
+ if brokerResult == nil {
+ rlog.Warning("no broker found for mq", map[string]interface{}{
+ rlog.LogKeyMessageQueue: queue,
+ })
+ return nil, ErrBrokerNotFound
+ }
+
+ if brokerResult.Slave {
+ sysFlag = clearCommitOffsetFlag(sysFlag)
+ }
+
+ if (data.ExpType == string(TAG)) && brokerResult.BrokerVersion < internal.V4_1_0 {
+ return nil, fmt.Errorf("the broker [%s, %v] does not upgrade to support for filter message by %v",
+ queue.BrokerName, brokerResult.BrokerVersion, data.ExpType)
+ }
+
+ pullRequest := &internal.PullMessageRequestHeader{
+ ConsumerGroup: dc.consumerGroup,
+ Topic: queue.Topic,
+ QueueId: int32(queue.QueueId),
+ QueueOffset: offset,
+ MaxMsgNums: int32(numbers),
+ SysFlag: sysFlag,
+ CommitOffset: commitOffsetValue,
+ // TODO: 和java对齐
+ SuspendTimeoutMillis: _BrokerSuspendMaxTime,
+ SubExpression: data.SubString,
+ // TODO: add subversion
+ ExpressionType: string(data.ExpType),
+ }
+
+ if data.ExpType == string(TAG) {
+ pullRequest.SubVersion = 0
+ } else {
+ pullRequest.SubVersion = data.SubVersion
+ }
+
+ // TODO: add computPullFromWhichFilterServer
+
+ return dc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
+}
+
+func (dc *defaultConsumer) processPullResult(mq *primitive.MessageQueue, result *primitive.PullResult, data *internal.SubscriptionData) {
+
+ dc.updatePullFromWhichNode(mq, result.SuggestWhichBrokerId)
+
+ switch result.Status {
+ case primitive.PullFound:
+ result.SetMessageExts(primitive.DecodeMessage(result.GetBody()))
+ msgs := result.GetMessageExts()
+
+ // filter message according to tags
+ msgListFilterAgain := msgs
+ if data.Tags.Len() > 0 && data.ClassFilterMode {
+ msgListFilterAgain = make([]*primitive.MessageExt, 0)
+ for _, msg := range msgs {
+ _, exist := data.Tags.Contains(msg.GetTags())
+ if exist {
+ msgListFilterAgain = append(msgListFilterAgain, msg)
+ }
+ }
+ }
+
+ // TODO: add filter message hook
+ for _, msg := range msgListFilterAgain {
+ traFlag, _ := strconv.ParseBool(msg.GetProperty(primitive.PropertyTransactionPrepared))
+ if traFlag {
+ msg.TransactionId = msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
+ }
+
+ msg.WithProperty(primitive.PropertyMinOffset, strconv.FormatInt(result.MinOffset, 10))
+ msg.WithProperty(primitive.PropertyMaxOffset, strconv.FormatInt(result.MaxOffset, 10))
+ }
+
+ result.SetMessageExts(msgListFilterAgain)
+ }
+}
+
+func (dc *defaultConsumer) findConsumerList(topic string) []string {
+ brokerAddr := dc.namesrv.FindBrokerAddrByTopic(topic)
+ if brokerAddr == "" {
+ dc.namesrv.UpdateTopicRouteInfo(topic)
+ brokerAddr = dc.namesrv.FindBrokerAddrByTopic(topic)
+ }
+
+ if brokerAddr != "" {
+ req := &internal.GetConsumerListRequestHeader{
+ ConsumerGroup: dc.consumerGroup,
+ }
+ cmd := remote.NewRemotingCommand(internal.ReqGetConsumerListByGroup, req, nil)
+ res, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second) // TODO 超时机制有问题
+ if err != nil {
+ rlog.Error("get consumer list of group from broker error", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: dc.consumerGroup,
+ rlog.LogKeyBroker: brokerAddr,
+ rlog.LogKeyUnderlayError: err,
+ })
+ return nil
+ }
+ result := gjson.ParseBytes(res.Body)
+ list := make([]string, 0)
+ arr := result.Get("consumerIdList").Array()
+ for idx := range arr {
+ list = append(list, arr[idx].String())
+ }
+ return list
+ }
+ return nil
+}
+
+func (dc *defaultConsumer) sendBack(msg *primitive.MessageExt, level int) error {
+ return nil
+}
+
+// QueryMaxOffset with specific queueId and topic
+func (dc *defaultConsumer) queryMaxOffset(mq *primitive.MessageQueue) (int64, error) {
+ brokerAddr := dc.namesrv.FindBrokerAddrByName(mq.BrokerName)
+ if brokerAddr == "" {
+ dc.namesrv.UpdateTopicRouteInfo(mq.Topic)
+ brokerAddr = dc.namesrv.FindBrokerAddrByName(mq.Topic)
+ }
+ if brokerAddr == "" {
+ return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
+ }
+
+ request := &internal.GetMaxOffsetRequestHeader{
+ Topic: mq.Topic,
+ QueueId: mq.QueueId,
+ }
+
+ cmd := remote.NewRemotingCommand(internal.ReqGetMaxOffset, request, nil)
+ response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
+ if err != nil {
+ return -1, err
+ }
+
+ return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
+}
+
+func (dc *defaultConsumer) queryOffset(mq *primitive.MessageQueue) int64 {
+ return dc.storage.read(mq, _ReadMemoryThenStore)
+}
+
+// SearchOffsetByTimestamp with specific queueId and topic
+func (dc *defaultConsumer) searchOffsetByTimestamp(mq *primitive.MessageQueue, timestamp int64) (int64, error) {
+ brokerAddr := dc.namesrv.FindBrokerAddrByName(mq.BrokerName)
+ if brokerAddr == "" {
+ dc.namesrv.UpdateTopicRouteInfo(mq.Topic)
+ brokerAddr = dc.namesrv.FindBrokerAddrByName(mq.Topic)
+ }
+ if brokerAddr == "" {
+ return -1, fmt.Errorf("the broker [%s] does not exist", mq.BrokerName)
+ }
+
+ request := &internal.SearchOffsetRequestHeader{
+ Topic: mq.Topic,
+ QueueId: mq.QueueId,
+ Timestamp: timestamp,
+ }
+
+ cmd := remote.NewRemotingCommand(internal.ReqSearchOffsetByTimestamp, request, nil)
+ response, err := dc.client.InvokeSync(context.Background(), brokerAddr, cmd, 3*time.Second)
+ if err != nil {
+ return -1, err
+ }
+
+ return strconv.ParseInt(response.ExtFields["offset"], 10, 64)
+}
+
+func buildSubscriptionData(topic string, selector MessageSelector) *internal.SubscriptionData {
+ subData := &internal.SubscriptionData{
+ Topic: topic,
+ SubString: selector.Expression,
+ ExpType: string(selector.Type),
+ }
+
+ if selector.Type != "" && selector.Type != TAG {
+ return subData
+ }
+
+ if selector.Expression == "" || selector.Expression == _SubAll {
+ subData.ExpType = string(TAG)
+ subData.SubString = _SubAll
+ } else {
+ tags := strings.Split(selector.Expression, "||")
+ subData.Tags = utils.NewSet()
+ subData.Codes = utils.NewSet()
+ for idx := range tags {
+ trimString := strings.Trim(tags[idx], " ")
+ if trimString != "" {
+ if _, ok := subData.Tags.Contains(trimString); !ok {
+ subData.Tags.AddKV(trimString, trimString)
+ }
+ hCode := utils.HashString(trimString)
+ v := strconv.Itoa(hCode)
+ if _, ok := subData.Codes.Contains(v); !ok {
+ subData.Codes.AddKV(v, v)
+ }
+ }
+ }
+ }
+ return subData
+}
+
+func buildSysFlag(commitOffset, suspend, subscription, classFilter bool) int32 {
+ var flag int32 = 0
+ if commitOffset {
+ flag |= 0x1 << 0
+ }
+
+ if suspend {
+ flag |= 0x1 << 1
+ }
+
+ if subscription {
+ flag |= 0x1 << 2
+ }
+
+ if classFilter {
+ flag |= 0x1 << 3
+ }
+
+ return flag
+}
+
+func clearCommitOffsetFlag(sysFlag int32) int32 {
+ return sysFlag & (^0x1 << 0)
+}
+
+func (dc *defaultConsumer) tryFindBroker(mq *primitive.MessageQueue) *internal.FindBrokerResult {
+ result := dc.namesrv.FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
+ if result != nil {
+ return result
+ }
+ dc.namesrv.UpdateTopicRouteInfo(mq.Topic)
+ return dc.namesrv.FindBrokerAddressInSubscribe(mq.BrokerName, dc.recalculatePullFromWhichNode(mq), false)
+}
+
+func (dc *defaultConsumer) updatePullFromWhichNode(mq *primitive.MessageQueue, brokerId int64) {
+ dc.pullFromWhichNodeTable.Store(*mq, brokerId)
+}
+
+func (dc *defaultConsumer) recalculatePullFromWhichNode(mq *primitive.MessageQueue) int64 {
+ v, exist := dc.pullFromWhichNodeTable.Load(*mq)
+ if exist {
+ return v.(int64)
+ }
+ return internal.MasterId
+}
diff --git a/consumer/consumer_test.go b/consumer/consumer_test.go
new file mode 100644
index 0000000..8b99767
--- /dev/null
+++ b/consumer/consumer_test.go
@@ -0,0 +1,170 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/golang/mock/gomock"
+ . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func TestParseTimestamp(t *testing.T) {
+ layout := "20060102150405"
+ timestamp, err := time.ParseInLocation(layout, "20190430193409", time.UTC)
+ assert.Nil(t, err)
+ assert.Equal(t, int64(1556652849), timestamp.Unix())
+}
+
+func TestDoRebalance(t *testing.T) {
+ Convey("Given a defaultConsumer", t, func() {
+ dc := &defaultConsumer{
+ model: Clustering,
+ }
+
+ topic := "test"
+ broker := "127.0.0.1:8889"
+ clientID := "clientID"
+ mqs := []*primitive.MessageQueue{
+ {
+ Topic: topic,
+ BrokerName: "",
+ QueueId: 0,
+ },
+ {
+ Topic: topic,
+ BrokerName: "",
+ QueueId: 1,
+ },
+ }
+ dc.topicSubscribeInfoTable.Store(topic, mqs)
+ sub := &internal.SubscriptionData{}
+ dc.subscriptionDataTable.Store(topic, sub)
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+ namesrvCli := internal.NewMockNamesrvs(ctrl)
+ namesrvCli.EXPECT().FindBrokerAddrByTopic(gomock.Any()).Return(broker)
+ dc.namesrv = namesrvCli
+
+ rmqCli := internal.NewMockRMQClient(ctrl)
+ rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+ Return(&remote.RemotingCommand{
+ Body: []byte("{\"consumerIdList\": [\"a1\", \"a2\", \"a3\"] }"),
+ }, nil)
+ rmqCli.EXPECT().ClientID().Return(clientID)
+ dc.client = rmqCli
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+ dc.allocate = func(cg string, clientID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
+ assert.Equal(t, cidAll, []string{"a1", "a2", "a3"})
+ wg.Done()
+ return nil
+ }
+
+ dc.doBalance()
+
+ wg.Wait()
+ })
+}
+
+func TestComputePullFromWhere(t *testing.T) {
+ Convey("Given a defaultConsumer", t, func() {
+ dc := &defaultConsumer{
+ model: Clustering,
+ cType: _PushConsume,
+ }
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+
+ offsetStore := NewMockOffsetStore(ctrl)
+ dc.storage = offsetStore
+
+ mq := &primitive.MessageQueue{
+ Topic: "test",
+ }
+
+ namesrvCli := internal.NewMockNamesrvs(ctrl)
+ dc.namesrv = namesrvCli
+
+ rmqCli := internal.NewMockRMQClient(ctrl)
+ dc.client = rmqCli
+
+ Convey("get effective offset", func() {
+ offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(10))
+ res := dc.computePullFromWhere(mq)
+ assert.Equal(t, int64(10), res)
+ })
+
+ Convey("ConsumeFromLastOffset for normal topic", func() {
+ offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
+ dc.option.FromWhere = ConsumeFromLastOffset
+
+ broker := "a"
+ namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
+
+ rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+ Return(&remote.RemotingCommand{
+ ExtFields: map[string]string{
+ "offset": "20",
+ },
+ }, nil)
+
+ res := dc.computePullFromWhere(mq)
+ assert.Equal(t, int64(20), res)
+ })
+
+ Convey("ConsumeFromFirstOffset for normal topic", func() {
+ offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
+ dc.option.FromWhere = ConsumeFromFirstOffset
+
+ res := dc.computePullFromWhere(mq)
+ assert.Equal(t, int64(0), res)
+ })
+
+ Convey("ConsumeFromTimestamp for normal topic", func() {
+ offsetStore.EXPECT().read(gomock.Any(), gomock.Any()).Return(int64(-1))
+ dc.option.FromWhere = ConsumeFromTimestamp
+
+ dc.option.ConsumeTimestamp = "20060102150405"
+
+ broker := "a"
+ namesrvCli.EXPECT().FindBrokerAddrByName(gomock.Any()).Return(broker)
+
+ rmqCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
+ Return(&remote.RemotingCommand{
+ ExtFields: map[string]string{
+ "offset": "30",
+ },
+ }, nil)
+
+ res := dc.computePullFromWhere(mq)
+ assert.Equal(t, int64(30), res)
+ })
+
+ })
+}
diff --git a/consumer/interceptor.go b/consumer/interceptor.go
new file mode 100644
index 0000000..aababfe
--- /dev/null
+++ b/consumer/interceptor.go
@@ -0,0 +1,109 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "context"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+// WithTrace support rocketmq trace: https://github.com/apache/rocketmq/wiki/RIP-6-Message-Trace.
+func WithTrace(traceCfg *primitive.TraceConfig) Option {
+ return func(options *consumerOptions) {
+
+ ori := options.Interceptors
+ options.Interceptors = make([]primitive.Interceptor, 0)
+ options.Interceptors = append(options.Interceptors, newTraceInterceptor(traceCfg))
+ options.Interceptors = append(options.Interceptors, ori...)
+ }
+}
+
+func newTraceInterceptor(traceCfg *primitive.TraceConfig) primitive.Interceptor {
+ dispatcher := internal.NewTraceDispatcher(traceCfg)
+ dispatcher.Start()
+
+ return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
+ consumerCtx, exist := primitive.GetConsumerCtx(ctx)
+ if !exist || len(consumerCtx.Msgs) == 0 {
+ return next(ctx, req, reply)
+ }
+
+ beginT := time.Now()
+ // before traceCtx
+ traceCx := internal.TraceContext{
+ RequestId: primitive.CreateUniqID(),
+ TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
+ TraceType: internal.SubBefore,
+ GroupName: consumerCtx.ConsumerGroup,
+ IsSuccess: true,
+ }
+ beans := make([]internal.TraceBean, 0)
+ for _, msg := range consumerCtx.Msgs {
+ if msg == nil {
+ continue
+ }
+ regionID := msg.GetRegionID()
+ traceOn := msg.IsTraceOn()
+ if traceOn == "false" {
+ continue
+ }
+ bean := internal.TraceBean{
+ Topic: msg.Topic,
+ MsgId: msg.MsgId,
+ Tags: msg.GetTags(),
+ Keys: msg.GetKeys(),
+ StoreTime: msg.StoreTimestamp,
+ BodyLength: int(msg.StoreSize),
+ RetryTimes: int(msg.ReconsumeTimes),
+ ClientHost: utils.LocalIP,
+ StoreHost: utils.LocalIP,
+ }
+ beans = append(beans, bean)
+ traceCx.RegionId = regionID
+ }
+ if len(beans) > 0 {
+ traceCx.TraceBeans = beans
+ traceCx.TimeStamp = time.Now().UnixNano() / int64(time.Millisecond)
+ dispatcher.Append(traceCx)
+ }
+
+ err := next(ctx, req, reply)
+
+ // after traceCtx
+ costTime := time.Since(beginT).Nanoseconds() / int64(time.Millisecond)
+ ctxType := consumerCtx.Properties[primitive.PropCtxType]
+ afterCtx := internal.TraceContext{
+ TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
+
+ TraceType: internal.SubAfter,
+ RegionId: traceCx.RegionId,
+ GroupName: traceCx.GroupName,
+ RequestId: traceCx.RequestId,
+ IsSuccess: consumerCtx.Success,
+ CostTime: costTime,
+ TraceBeans: traceCx.TraceBeans,
+ ContextCode: primitive.ConsumeReturnType(ctxType).Ordinal(),
+ }
+ dispatcher.Append(afterCtx)
+ return err
+ }
+}
diff --git a/consumer/lock.go b/consumer/lock.go
new file mode 100644
index 0000000..6fb17cd
--- /dev/null
+++ b/consumer/lock.go
@@ -0,0 +1,37 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "sync"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+type QueueLock struct {
+ lockTable sync.Map
+}
+
+func newQueueLock() *QueueLock {
+ return &QueueLock{}
+}
+
+func (ql QueueLock) fetchLock(queue primitive.MessageQueue) sync.Locker {
+ v, _ := ql.lockTable.LoadOrStore(queue, new(sync.Mutex))
+ return v.(*sync.Mutex)
+}
diff --git a/consumer/mock_offset_store.go b/consumer/mock_offset_store.go
new file mode 100644
index 0000000..bac1cdb
--- /dev/null
+++ b/consumer/mock_offset_store.go
@@ -0,0 +1,94 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by MockGen. DO NOT EDIT.
+// Source: offset_store.go
+
+// Package consumer is a generated GoMock package.
+package consumer
+
+import (
+ reflect "reflect"
+
+ primitive "github.com/apache/rocketmq-client-go/v2/primitive"
+ gomock "github.com/golang/mock/gomock"
+)
+
+// MockOffsetStore is a mock of OffsetStore interface
+type MockOffsetStore struct {
+ ctrl *gomock.Controller
+ recorder *MockOffsetStoreMockRecorder
+}
+
+// MockOffsetStoreMockRecorder is the mock recorder for MockOffsetStore
+type MockOffsetStoreMockRecorder struct {
+ mock *MockOffsetStore
+}
+
+// NewMockOffsetStore creates a new mock instance
+func NewMockOffsetStore(ctrl *gomock.Controller) *MockOffsetStore {
+ mock := &MockOffsetStore{ctrl: ctrl}
+ mock.recorder = &MockOffsetStoreMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockOffsetStore) EXPECT() *MockOffsetStoreMockRecorder {
+ return m.recorder
+}
+
+// persist mocks base method
+func (m *MockOffsetStore) persist(mqs []*primitive.MessageQueue) {
+ m.ctrl.Call(m, "persist", mqs)
+}
+
+// persist indicates an expected call of persist
+func (mr *MockOffsetStoreMockRecorder) persist(mqs interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "persist", reflect.TypeOf((*MockOffsetStore)(nil).persist), mqs)
+}
+
+// remove mocks base method
+func (m *MockOffsetStore) remove(mq *primitive.MessageQueue) {
+ m.ctrl.Call(m, "remove", mq)
+}
+
+// remove indicates an expected call of remove
+func (mr *MockOffsetStoreMockRecorder) remove(mq interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "remove", reflect.TypeOf((*MockOffsetStore)(nil).remove), mq)
+}
+
+// read mocks base method
+func (m *MockOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
+ ret := m.ctrl.Call(m, "read", mq, t)
+ ret0, _ := ret[0].(int64)
+ return ret0
+}
+
+// read indicates an expected call of read
+func (mr *MockOffsetStoreMockRecorder) read(mq, t interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "read", reflect.TypeOf((*MockOffsetStore)(nil).read), mq, t)
+}
+
+// update mocks base method
+func (m *MockOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
+ m.ctrl.Call(m, "update", mq, offset, increaseOnly)
+}
+
+// update indicates an expected call of update
+func (mr *MockOffsetStoreMockRecorder) update(mq, offset, increaseOnly interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "update", reflect.TypeOf((*MockOffsetStore)(nil).update), mq, offset, increaseOnly)
+}
diff --git a/consumer/offset_store.go b/consumer/offset_store.go
new file mode 100644
index 0000000..44a0597
--- /dev/null
+++ b/consumer/offset_store.go
@@ -0,0 +1,381 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "time"
+
+ jsoniter "github.com/json-iterator/go"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+type readType int
+
+const (
+ _ReadFromMemory readType = iota
+ _ReadFromStore
+ _ReadMemoryThenStore
+)
+
+var (
+ _LocalOffsetStorePath = os.Getenv("rocketmq.client.localOffsetStoreDir")
+)
+
+func init() {
+ if _LocalOffsetStorePath == "" {
+ _LocalOffsetStorePath = filepath.Join(os.Getenv("HOME"), ".rocketmq_client_go")
+ }
+}
+
+//go:generate mockgen -source offset_store.go -destination mock_offset_store.go -self_package github.com/apache/rocketmq-client-go/v2/consumer --package consumer OffsetStore
+type OffsetStore interface {
+ persist(mqs []*primitive.MessageQueue)
+ remove(mq *primitive.MessageQueue)
+ read(mq *primitive.MessageQueue, t readType) int64
+ update(mq *primitive.MessageQueue, offset int64, increaseOnly bool)
+}
+
+type OffsetSerializeWrapper struct {
+ OffsetTable map[MessageQueueKey]int64 `json:"offsetTable"`
+}
+
+type MessageQueueKey primitive.MessageQueue
+
+func (mq MessageQueueKey) MarshalText() (text []byte, err error) {
+ repr := struct {
+ Topic string `json:"topic"`
+ BrokerName string `json:"brokerName"`
+ QueueId int `json:"queueId"`
+ }{
+ Topic: mq.Topic,
+ BrokerName: mq.BrokerName,
+ QueueId: mq.QueueId,
+ }
+ text, err = jsoniter.Marshal(repr)
+ return
+}
+
+func (mq *MessageQueueKey) UnmarshalText(text []byte) error {
+ repr := struct {
+ Topic string `json:"topic"`
+ BrokerName string `json:"brokerName"`
+ QueueId int `json:"queueId"`
+ }{}
+ err := jsoniter.Unmarshal(text, &repr)
+ if err != nil {
+ return err
+ }
+ mq.Topic = repr.Topic
+ mq.QueueId = repr.QueueId
+ mq.BrokerName = repr.BrokerName
+
+ return nil
+}
+
+type localFileOffsetStore struct {
+ group string
+ path string
+ OffsetTable map[MessageQueueKey]int64
+ // mutex for offset file
+ mutex sync.Mutex
+}
+
+func NewLocalFileOffsetStore(clientID, group string) OffsetStore {
+ store := &localFileOffsetStore{
+ group: group,
+ path: filepath.Join(_LocalOffsetStorePath, clientID, group, "offset.json"),
+ OffsetTable: make(map[MessageQueueKey]int64),
+ }
+ store.load()
+ return store
+}
+
+func (local *localFileOffsetStore) load() {
+ local.mutex.Lock()
+ defer local.mutex.Unlock()
+ data, err := utils.FileReadAll(local.path)
+ if os.IsNotExist(err) {
+ return
+ }
+ if err != nil {
+ rlog.Info("read from local store error, try to use bak file", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ data, err = utils.FileReadAll(filepath.Join(local.path, ".bak"))
+ }
+ if err != nil {
+ rlog.Info("read from local store bak file error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ return
+ }
+ datas := make(map[MessageQueueKey]int64)
+
+ wrapper := OffsetSerializeWrapper{
+ OffsetTable: datas,
+ }
+
+ err = jsoniter.Unmarshal(data, &wrapper)
+ if err != nil {
+ rlog.Warning("unmarshal local offset error", map[string]interface{}{
+ "local_path": local.path,
+ rlog.LogKeyUnderlayError: err.Error(),
+ })
+ return
+ }
+
+ if datas != nil {
+ local.OffsetTable = datas
+ }
+}
+
+func (local *localFileOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
+ switch t {
+ case _ReadFromMemory, _ReadMemoryThenStore:
+ off := readFromMemory(local.OffsetTable, mq)
+ if off >= 0 || (off == -1 && t == _ReadFromMemory) {
+ return off
+ }
+ fallthrough
+ case _ReadFromStore:
+ local.load()
+ return readFromMemory(local.OffsetTable, mq)
+ default:
+
+ }
+ return -1
+}
+
+func (local *localFileOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
+ local.mutex.Lock()
+ defer local.mutex.Unlock()
+ rlog.Debug("update offset", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq,
+ "new_offset": offset,
+ })
+ key := MessageQueueKey(*mq)
+ localOffset, exist := local.OffsetTable[key]
+ if !exist {
+ local.OffsetTable[key] = offset
+ return
+ }
+ if increaseOnly {
+ if localOffset < offset {
+ local.OffsetTable[key] = offset
+ }
+ } else {
+ local.OffsetTable[key] = offset
+ }
+}
+
+func (local *localFileOffsetStore) persist(mqs []*primitive.MessageQueue) {
+ if len(mqs) == 0 {
+ return
+ }
+ local.mutex.Lock()
+ defer local.mutex.Unlock()
+
+ wrapper := OffsetSerializeWrapper{
+ OffsetTable: local.OffsetTable,
+ }
+
+ data, _ := jsoniter.Marshal(wrapper)
+ utils.CheckError(fmt.Sprintf("persist offset to %s", local.path), utils.WriteToFile(local.path, data))
+}
+
+func (local *localFileOffsetStore) remove(mq *primitive.MessageQueue) {
+ // nothing to do
+}
+
+type remoteBrokerOffsetStore struct {
+ group string
+ OffsetTable map[primitive.MessageQueue]int64 `json:"OffsetTable"`
+ client internal.RMQClient
+ namesrv internal.Namesrvs
+ mutex sync.RWMutex
+}
+
+func NewRemoteOffsetStore(group string, client internal.RMQClient, namesrv internal.Namesrvs) OffsetStore {
+ return &remoteBrokerOffsetStore{
+ group: group,
+ client: client,
+ namesrv: namesrv,
+ OffsetTable: make(map[primitive.MessageQueue]int64),
+ }
+}
+
+func (r *remoteBrokerOffsetStore) persist(mqs []*primitive.MessageQueue) {
+ r.mutex.Lock()
+ defer r.mutex.Unlock()
+ if len(mqs) == 0 {
+ return
+ }
+
+ used := make(map[primitive.MessageQueue]struct{}, 0)
+ for _, mq := range mqs {
+ used[*mq] = struct{}{}
+ }
+
+ for mq, off := range r.OffsetTable {
+ if _, ok := used[mq]; !ok {
+ delete(r.OffsetTable, mq)
+ continue
+ }
+ err := r.updateConsumeOffsetToBroker(r.group, mq, off)
+ if err != nil {
+ rlog.Warning("update offset to broker error", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: r.group,
+ rlog.LogKeyMessageQueue: mq.String(),
+ rlog.LogKeyUnderlayError: err.Error(),
+ "offset": off,
+ })
+ }
+ }
+}
+
+func (r *remoteBrokerOffsetStore) remove(mq *primitive.MessageQueue) {
+ r.mutex.Lock()
+ defer r.mutex.Unlock()
+
+ delete(r.OffsetTable, *mq)
+ rlog.Info("delete mq from offset table", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq,
+ })
+}
+
+func (r *remoteBrokerOffsetStore) read(mq *primitive.MessageQueue, t readType) int64 {
+ r.mutex.RLock()
+ switch t {
+ case _ReadFromMemory, _ReadMemoryThenStore:
+ off, exist := r.OffsetTable[*mq]
+ if exist {
+ r.mutex.RUnlock()
+ return off
+ }
+ if t == _ReadFromMemory {
+ r.mutex.RUnlock()
+ return -1
+ }
+ fallthrough
+ case _ReadFromStore:
+ off, err := r.fetchConsumeOffsetFromBroker(r.group, mq)
+ if err != nil {
+ rlog.Error("fecth offset of mq error", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq.String(),
+ rlog.LogKeyUnderlayError: err,
+ })
+ r.mutex.RUnlock()
+ return -1
+ }
+ r.mutex.RUnlock()
+ r.update(mq, off, true)
+ return off
+ default:
+ }
+
+ return -1
+}
+
+func (r *remoteBrokerOffsetStore) update(mq *primitive.MessageQueue, offset int64, increaseOnly bool) {
+ r.mutex.Lock()
+ defer r.mutex.Unlock()
+ localOffset, exist := r.OffsetTable[*mq]
+ if !exist {
+ r.OffsetTable[*mq] = offset
+ return
+ }
+ if increaseOnly {
+ if localOffset < offset {
+ r.OffsetTable[*mq] = offset
+ }
+ } else {
+ r.OffsetTable[*mq] = offset
+ }
+}
+
+func (r *remoteBrokerOffsetStore) fetchConsumeOffsetFromBroker(group string, mq *primitive.MessageQueue) (int64, error) {
+ broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
+ if broker == "" {
+ r.namesrv.UpdateTopicRouteInfo(mq.Topic)
+ broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
+ }
+ if broker == "" {
+ return int64(-1), fmt.Errorf("broker: %s address not found", mq.BrokerName)
+ }
+ queryOffsetRequest := &internal.QueryConsumerOffsetRequestHeader{
+ ConsumerGroup: group,
+ Topic: mq.Topic,
+ QueueId: mq.QueueId,
+ }
+ cmd := remote.NewRemotingCommand(internal.ReqQueryConsumerOffset, queryOffsetRequest, nil)
+ res, err := r.client.InvokeSync(context.Background(), broker, cmd, 3*time.Second)
+ if err != nil {
+ return -1, err
+ }
+ if res.Code != internal.ResSuccess {
+ return -2, fmt.Errorf("broker response code: %d, remarks: %s", res.Code, res.Remark)
+ }
+
+ off, err := strconv.ParseInt(res.ExtFields["offset"], 10, 64)
+
+ if err != nil {
+ return -1, err
+ }
+
+ return off, nil
+}
+
+func (r *remoteBrokerOffsetStore) updateConsumeOffsetToBroker(group string, mq primitive.MessageQueue, off int64) error {
+ broker := r.namesrv.FindBrokerAddrByName(mq.BrokerName)
+ if broker == "" {
+ r.namesrv.UpdateTopicRouteInfo(mq.Topic)
+ broker = r.namesrv.FindBrokerAddrByName(mq.BrokerName)
+ }
+ if broker == "" {
+ return fmt.Errorf("broker: %s address not found", mq.BrokerName)
+ }
+
+ updateOffsetRequest := &internal.UpdateConsumerOffsetRequestHeader{
+ ConsumerGroup: group,
+ Topic: mq.Topic,
+ QueueId: mq.QueueId,
+ CommitOffset: off,
+ }
+ cmd := remote.NewRemotingCommand(internal.ReqUpdateConsumerOffset, updateOffsetRequest, nil)
+ return r.client.InvokeOneWay(context.Background(), broker, cmd, 5*time.Second)
+}
+
+func readFromMemory(table map[MessageQueueKey]int64, mq *primitive.MessageQueue) int64 {
+ localOffset, exist := table[MessageQueueKey(*mq)]
+ if !exist {
+ return -1
+ }
+
+ return localOffset
+}
diff --git a/consumer/offset_store_test.go b/consumer/offset_store_test.go
new file mode 100644
index 0000000..27b98d9
--- /dev/null
+++ b/consumer/offset_store_test.go
@@ -0,0 +1,244 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/golang/mock/gomock"
+ . "github.com/smartystreets/goconvey/convey"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func TestNewLocalFileOffsetStore(t *testing.T) {
+ Convey("Given some test cases", t, func() {
+ type testCase struct {
+ clientId string
+ group string
+ expectedResult *localFileOffsetStore
+ }
+ cases := []testCase{
+ {
+ clientId: "",
+ group: "testGroup",
+ expectedResult: &localFileOffsetStore{
+ group: "testGroup",
+ path: filepath.Join(_LocalOffsetStorePath, "/testGroup/offset.json"),
+ },
+ }, {
+ clientId: "192.168.24.1@default",
+ group: "",
+ expectedResult: &localFileOffsetStore{
+ group: "",
+ path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/offset.json"),
+ },
+ }, {
+ clientId: "192.168.24.1@default",
+ group: "testGroup",
+ expectedResult: &localFileOffsetStore{
+ group: "testGroup",
+ path: filepath.Join(_LocalOffsetStorePath, "/192.168.24.1@default/testGroup/offset.json"),
+ },
+ },
+ }
+
+ for _, value := range cases {
+ result := NewLocalFileOffsetStore(value.clientId, value.group).(*localFileOffsetStore)
+ value.expectedResult.OffsetTable = result.OffsetTable
+ So(result, ShouldResemble, value.expectedResult)
+ }
+ })
+}
+
+func TestLocalFileOffsetStore(t *testing.T) {
+ Convey("Given a local store with a starting value", t, func() {
+ localStore := NewLocalFileOffsetStore("192.168.24.1@default", "testGroup")
+
+ type offsetCase struct {
+ queue *primitive.MessageQueue
+ setOffset int64
+ expectedOffset int64
+ }
+ mq := &primitive.MessageQueue{
+ Topic: "testTopic",
+ BrokerName: "default",
+ QueueId: 1,
+ }
+
+ Convey("test update", func() {
+ Convey("when increaseOnly is false", func() {
+ cases := []offsetCase{
+ {
+ queue: mq,
+ setOffset: 3,
+ expectedOffset: 3,
+ }, {
+ queue: mq,
+ setOffset: 1,
+ expectedOffset: 1,
+ },
+ }
+ for _, value := range cases {
+ localStore.update(value.queue, value.setOffset, false)
+ offset := localStore.read(value.queue, _ReadFromMemory)
+ So(offset, ShouldEqual, value.expectedOffset)
+ }
+ })
+
+ Convey("when increaseOnly is true", func() {
+ localStore.update(mq, 0, false)
+
+ cases := []offsetCase{
+ {
+ queue: mq,
+ setOffset: 3,
+ expectedOffset: 3,
+ }, {
+ queue: mq,
+ setOffset: 1,
+ expectedOffset: 3,
+ },
+ }
+ for _, value := range cases {
+ localStore.update(value.queue, value.setOffset, true)
+ offset := localStore.read(value.queue, _ReadFromMemory)
+ So(offset, ShouldEqual, value.expectedOffset)
+ }
+ })
+ })
+
+ Convey("test persist", func() {
+ localStore.update(mq, 1, false)
+ offset := localStore.read(mq, _ReadFromMemory)
+ So(offset, ShouldEqual, 1)
+
+ queues := []*primitive.MessageQueue{mq}
+ localStore.persist(queues)
+ offset = localStore.read(mq, _ReadFromStore)
+ So(offset, ShouldEqual, 1)
+
+ delete(localStore.(*localFileOffsetStore).OffsetTable, MessageQueueKey(*mq))
+ offset = localStore.read(mq, _ReadMemoryThenStore)
+ So(offset, ShouldEqual, 1)
+ })
+ })
+}
+
+func TestRemoteBrokerOffsetStore(t *testing.T) {
+ Convey("Given a remote store with a starting value", t, func() {
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+
+ namesrv := internal.NewMockNamesrvs(ctrl)
+
+ rmqClient := internal.NewMockRMQClient(ctrl)
+ remoteStore := NewRemoteOffsetStore("testGroup", rmqClient, namesrv)
+
+ type offsetCase struct {
+ queue *primitive.MessageQueue
+ setOffset int64
+ expectedOffset int64
+ }
+ mq := &primitive.MessageQueue{
+ Topic: "testTopic",
+ BrokerName: "default",
+ QueueId: 1,
+ }
+
+ Convey("test update", func() {
+ Convey("when increaseOnly is false", func() {
+ cases := []offsetCase{
+ {
+ queue: mq,
+ setOffset: 3,
+ expectedOffset: 3,
+ }, {
+ queue: mq,
+ setOffset: 1,
+ expectedOffset: 1,
+ },
+ }
+ for _, value := range cases {
+ remoteStore.update(value.queue, value.setOffset, false)
+ offset := remoteStore.read(value.queue, _ReadFromMemory)
+ So(offset, ShouldEqual, value.expectedOffset)
+ }
+ })
+
+ Convey("when increaseOnly is true", func() {
+ remoteStore.update(mq, 0, false)
+
+ cases := []offsetCase{
+ {
+ queue: mq,
+ setOffset: 3,
+ expectedOffset: 3,
+ }, {
+ queue: mq,
+ setOffset: 1,
+ expectedOffset: 3,
+ },
+ }
+ for _, value := range cases {
+ remoteStore.update(value.queue, value.setOffset, true)
+ offset := remoteStore.read(value.queue, _ReadFromMemory)
+ So(offset, ShouldEqual, value.expectedOffset)
+ }
+ })
+ })
+
+ Convey("test persist", func() {
+ queues := []*primitive.MessageQueue{mq}
+
+ namesrv.EXPECT().FindBrokerAddrByName(gomock.Any()).Return("192.168.24.1:10911").MaxTimes(2)
+
+ ret := &remote.RemotingCommand{
+ Code: internal.ResSuccess,
+ ExtFields: map[string]string{
+ "offset": "1",
+ },
+ }
+ rmqClient.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(ret, nil).MaxTimes(2)
+
+ remoteStore.persist(queues)
+ offset := remoteStore.read(mq, _ReadFromStore)
+ So(offset, ShouldEqual, 1)
+
+ remoteStore.remove(mq)
+ offset = remoteStore.read(mq, _ReadFromMemory)
+ So(offset, ShouldEqual, -1)
+ offset = remoteStore.read(mq, _ReadMemoryThenStore)
+ So(offset, ShouldEqual, 1)
+
+ })
+
+ Convey("test remove", func() {
+ remoteStore.update(mq, 1, false)
+ offset := remoteStore.read(mq, _ReadFromMemory)
+ So(offset, ShouldEqual, 1)
+
+ remoteStore.remove(mq)
+ offset = remoteStore.read(mq, _ReadFromMemory)
+ So(offset, ShouldEqual, -1)
+ })
+ })
+}
diff --git a/consumer/option.go b/consumer/option.go
new file mode 100644
index 0000000..07b4246
--- /dev/null
+++ b/consumer/option.go
@@ -0,0 +1,276 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+type consumerOptions struct {
+ internal.ClientOptions
+
+ /**
+ * Backtracking consumption time with second precision. Time format is
+ * 20131223171201<br>
+ * Implying Seventeen twelve and 01 seconds on December 23, 2013 year<br>
+ * Default backtracking consumption time Half an hour ago.
+ */
+ ConsumeTimestamp string
+
+ // The socket timeout in milliseconds
+ ConsumerPullTimeout time.Duration
+
+ // Concurrently max span offset.it has no effect on sequential consumption
+ ConsumeConcurrentlyMaxSpan int
+
+ // Flow control threshold on queue level, each message queue will cache at most 1000 messages by default,
+ // Consider the {PullBatchSize}, the instantaneous value may exceed the limit
+ PullThresholdForQueue int64
+
+ // Limit the cached message size on queue level, each message queue will cache at most 100 MiB messages by default,
+ // Consider the {@code pullBatchSize}, the instantaneous value may exceed the limit
+ //
+ // The size of a message only measured by message body, so it's not accurate
+ PullThresholdSizeForQueue int
+
+ // Flow control threshold on topic level, default value is -1(Unlimited)
+ //
+ // The value of {@code pullThresholdForQueue} will be overwrote and calculated based on
+ // {@code pullThresholdForTopic} if it is't unlimited
+ //
+ // For example, if the value of pullThresholdForTopic is 1000 and 10 message queues are assigned to this consumer,
+ // then pullThresholdForQueue will be set to 100
+ PullThresholdForTopic int
+
+ // Limit the cached message size on topic level, default value is -1 MiB(Unlimited)
+ //
+ // The value of {@code pullThresholdSizeForQueue} will be overwrote and calculated based on
+ // {@code pullThresholdSizeForTopic} if it is't unlimited
+ //
+ // For example, if the value of pullThresholdSizeForTopic is 1000 MiB and 10 message queues are
+ // assigned to this consumer, then pullThresholdSizeForQueue will be set to 100 MiB
+ PullThresholdSizeForTopic int
+
+ // Message pull Interval
+ PullInterval time.Duration
+
+ // Batch consumption size
+ ConsumeMessageBatchMaxSize int
+
+ // Batch pull size
+ PullBatchSize int32
+
+ // Whether update subscription relationship when every pull
+ PostSubscriptionWhenPull bool
+
+ // Max re-consume times. -1 means 16 times.
+ //
+ // If messages are re-consumed more than {@link #maxReconsumeTimes} before Success, it's be directed to a deletion
+ // queue waiting.
+ MaxReconsumeTimes int32
+
+ // Suspending pulling time for cases requiring slow pulling like flow-control scenario.
+ SuspendCurrentQueueTimeMillis time.Duration
+
+ // Maximum amount of time a message may block the consuming thread.
+ ConsumeTimeout time.Duration
+
+ ConsumerModel MessageModel
+ Strategy AllocateStrategy
+ ConsumeOrderly bool
+ FromWhere ConsumeFromWhere
+
+ Interceptors []primitive.Interceptor
+ // TODO traceDispatcher
+ MaxTimeConsumeContinuously time.Duration
+ //
+ AutoCommit bool
+ RebalanceLockInterval time.Duration
+
+ Resolver primitive.NsResolver
+}
+
+func defaultPushConsumerOptions() consumerOptions {
+ opts := consumerOptions{
+ ClientOptions: internal.DefaultClientOptions(),
+ Strategy: AllocateByAveragely,
+ MaxTimeConsumeContinuously: time.Duration(60 * time.Second),
+ RebalanceLockInterval: 20 * time.Second,
+ MaxReconsumeTimes: -1,
+ ConsumerModel: Clustering,
+ AutoCommit: true,
+ Resolver: primitive.NewHttpResolver("DEFAULT"),
+ }
+ opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
+ return opts
+}
+
+type Option func(*consumerOptions)
+
+func defaultPullConsumerOptions() consumerOptions {
+ opts := consumerOptions{
+ ClientOptions: internal.DefaultClientOptions(),
+ Resolver: primitive.NewHttpResolver("DEFAULT"),
+ }
+ opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
+ return opts
+}
+
+func WithConsumerModel(m MessageModel) Option {
+ return func(options *consumerOptions) {
+ options.ConsumerModel = m
+ }
+}
+
+func WithConsumeFromWhere(w ConsumeFromWhere) Option {
+ return func(options *consumerOptions) {
+ options.FromWhere = w
+ }
+}
+
+func WithConsumerOrder(order bool) Option {
+ return func(options *consumerOptions) {
+ options.ConsumeOrderly = order
+ }
+}
+
+func WithConsumeMessageBatchMaxSize(consumeMessageBatchMaxSize int) Option {
+ return func(options *consumerOptions) {
+ options.ConsumeMessageBatchMaxSize = consumeMessageBatchMaxSize
+ }
+}
+
+// WithChainConsumerInterceptor returns a ConsumerOption that specifies the chained interceptor for consumer.
+// The first interceptor will be the outer most, while the last interceptor will be the inner most wrapper
+// around the real call.
+func WithInterceptor(fs ...primitive.Interceptor) Option {
+ return func(options *consumerOptions) {
+ options.Interceptors = append(options.Interceptors, fs...)
+ }
+}
+
+// WithGroupName set group name address
+func WithGroupName(group string) Option {
+ return func(opts *consumerOptions) {
+ if group == "" {
+ return
+ }
+ opts.GroupName = group
+ }
+}
+
+func WithInstance(name string) Option {
+ return func(options *consumerOptions) {
+ options.InstanceName = name
+ }
+}
+
+// WithNamespace set the namespace of consumer
+func WithNamespace(namespace string) Option {
+ return func(opts *consumerOptions) {
+ opts.Namespace = namespace
+ }
+}
+
+func WithVIPChannel(enable bool) Option {
+ return func(opts *consumerOptions) {
+ opts.VIPChannelEnabled = enable
+ }
+}
+
+// WithRetry return a Option that specifies the retry times when send failed.
+// TODO: use retry middleware instead
+func WithRetry(retries int) Option {
+ return func(opts *consumerOptions) {
+ opts.RetryTimes = retries
+ }
+}
+
+func WithCredentials(c primitive.Credentials) Option {
+ return func(options *consumerOptions) {
+ options.ClientOptions.Credentials = c
+ }
+}
+
+// WithMaxReconsumeTimes set MaxReconsumeTimes of options, if message reconsume greater than MaxReconsumeTimes, it will
+// be sent to retry or dlq topic. more info reference by examples/consumer/retry.
+func WithMaxReconsumeTimes(times int32) Option {
+ return func(opts *consumerOptions) {
+ opts.MaxReconsumeTimes = times
+ }
+}
+
+func WithStrategy(strategy AllocateStrategy) Option {
+ return func(opts *consumerOptions) {
+ opts.Strategy = strategy
+ }
+}
+
+func WithPullBatchSize(batchSize int32) Option {
+ return func(options *consumerOptions) {
+ options.PullBatchSize = batchSize
+ }
+}
+
+func WithRebalanceLockInterval(interval time.Duration) Option {
+ return func(options *consumerOptions) {
+ options.RebalanceLockInterval = interval
+ }
+}
+
+func WithAutoCommit(auto bool) Option {
+ return func(options *consumerOptions) {
+ options.AutoCommit = auto
+ }
+}
+
+func WithSuspendCurrentQueueTimeMillis(suspendT time.Duration) Option {
+ return func(options *consumerOptions) {
+ options.SuspendCurrentQueueTimeMillis = suspendT
+ }
+}
+
+func WithPullInterval(interval time.Duration) Option {
+ return func(options *consumerOptions) {
+ options.PullInterval = interval
+ }
+}
+
+// WithNsResovler set nameserver resolver to fetch nameserver addr
+func WithNsResovler(resolver primitive.NsResolver) Option {
+ return func(options *consumerOptions) {
+ options.Resolver = resolver
+ }
+}
+
+// WithNameServer set NameServer address, only support one NameServer cluster in alpha2
+func WithNameServer(nameServers primitive.NamesrvAddr) Option {
+ return func(options *consumerOptions) {
+ options.Resolver = primitive.NewPassthroughResolver(nameServers)
+ }
+}
+
+// WithNameServerDomain set NameServer domain
+func WithNameServerDomain(nameServerUrl string) Option {
+ return func(opts *consumerOptions) {
+ opts.Resolver = primitive.NewHttpResolver("DEFAULT", nameServerUrl)
+ }
+}
diff --git a/consumer/process_queue.go b/consumer/process_queue.go
new file mode 100644
index 0000000..aab3522
--- /dev/null
+++ b/consumer/process_queue.go
@@ -0,0 +1,377 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/emirpasic/gods/maps/treemap"
+ "github.com/emirpasic/gods/utils"
+ gods_util "github.com/emirpasic/gods/utils"
+ uatomic "go.uber.org/atomic"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+const (
+ _RebalanceLockMaxTime = 30 * time.Second
+ _RebalanceInterval = 20 * time.Second
+ _PullMaxIdleTime = 120 * time.Second
+)
+
+type processQueue struct {
+ cachedMsgCount int64
+ cachedMsgSize int64
+ tryUnlockTimes int64
+ queueOffsetMax int64
+ msgAccCnt int64
+ msgCache *treemap.Map
+ mutex sync.RWMutex
+ consumeLock sync.Mutex
+ consumingMsgOrderlyTreeMap *treemap.Map
+ dropped *uatomic.Bool
+ lastPullTime time.Time
+ lastConsumeTime atomic.Value
+ locked *uatomic.Bool
+ lastLockTime atomic.Value
+ consuming bool
+ lockConsume sync.Mutex
+ msgCh chan []*primitive.MessageExt
+ order bool
+}
+
+func newProcessQueue(order bool) *processQueue {
+ consumingMsgOrderlyTreeMap := treemap.NewWith(gods_util.Int64Comparator)
+
+ lastConsumeTime := atomic.Value{}
+ lastConsumeTime.Store(time.Now())
+
+ lastLockTime := atomic.Value{}
+ lastLockTime.Store(time.Now())
+
+ pq := &processQueue{
+ msgCache: treemap.NewWith(utils.Int64Comparator),
+ lastPullTime: time.Now(),
+ lastConsumeTime: lastConsumeTime,
+ lastLockTime: lastLockTime,
+ msgCh: make(chan []*primitive.MessageExt, 32),
+ consumingMsgOrderlyTreeMap: consumingMsgOrderlyTreeMap,
+ order: order,
+ locked: uatomic.NewBool(false),
+ dropped: uatomic.NewBool(false),
+ }
+ return pq
+}
+
+func (pq *processQueue) putMessage(messages ...*primitive.MessageExt) {
+ if len(messages) == 0 {
+ return
+ }
+ pq.mutex.Lock()
+ if !pq.order {
+ pq.msgCh <- messages
+ }
+ validMessageCount := 0
+ for idx := range messages {
+ msg := messages[idx]
+ _, found := pq.msgCache.Get(msg.QueueOffset)
+ if found {
+ continue
+ }
+ pq.msgCache.Put(msg.QueueOffset, msg)
+ validMessageCount++
+ pq.queueOffsetMax = msg.QueueOffset
+ atomic.AddInt64(&pq.cachedMsgSize, int64(len(msg.Body)))
+ }
+ pq.mutex.Unlock()
+
+ atomic.AddInt64(&pq.cachedMsgCount, int64(validMessageCount))
+
+ if pq.msgCache.Size() > 0 && !pq.consuming {
+ pq.consuming = true
+ }
+
+ msg := messages[len(messages)-1]
+ maxOffset, err := strconv.ParseInt(msg.GetProperty(primitive.PropertyMaxOffset), 10, 64)
+ if err != nil {
+ acc := maxOffset - msg.QueueOffset
+ if acc > 0 {
+ pq.msgAccCnt = acc
+ }
+ }
+}
+
+func (pq *processQueue) WithLock(lock bool) {
+ pq.locked.Store(lock)
+}
+
+func (pq *processQueue) IsLock() bool {
+ return pq.locked.Load()
+}
+
+func (pq *processQueue) WithDropped(dropped bool) {
+ pq.dropped.Store(dropped)
+}
+
+func (pq *processQueue) IsDroppd() bool {
+ return pq.dropped.Load()
+}
+
+func (pq *processQueue) UpdateLastConsumeTime() {
+ pq.lastConsumeTime.Store(time.Now())
+}
+
+func (pq *processQueue) LastConsumeTime() time.Time {
+ return pq.lastConsumeTime.Load().(time.Time)
+}
+
+func (pq *processQueue) UpdateLastLockTime() {
+ pq.lastLockTime.Store(time.Now())
+}
+
+func (pq *processQueue) LastLockTime() time.Time {
+ return pq.lastLockTime.Load().(time.Time)
+}
+
+func (pq *processQueue) makeMessageToCosumeAgain(messages ...*primitive.MessageExt) {
+ pq.mutex.Lock()
+ for _, msg := range messages {
+ pq.consumingMsgOrderlyTreeMap.Remove(msg.QueueOffset)
+ pq.msgCache.Put(msg.QueueOffset, msg)
+ }
+
+ pq.mutex.Unlock()
+}
+
+func (pq *processQueue) removeMessage(messages ...*primitive.MessageExt) int64 {
+ result := int64(-1)
+ pq.mutex.Lock()
+ pq.UpdateLastConsumeTime()
+ if !pq.msgCache.Empty() {
+ result = pq.queueOffsetMax + 1
+ removedCount := 0
+ for idx := range messages {
+ msg := messages[idx]
+ _, found := pq.msgCache.Get(msg.QueueOffset)
+ if !found {
+ continue
+ }
+ pq.msgCache.Remove(msg.QueueOffset)
+ removedCount++
+ atomic.AddInt64(&pq.cachedMsgSize, int64(-len(msg.Body)))
+ }
+ atomic.AddInt64(&pq.cachedMsgCount, int64(-removedCount))
+ }
+ if !pq.msgCache.Empty() {
+ first, _ := pq.msgCache.Min()
+ result = first.(int64)
+ }
+ pq.mutex.Unlock()
+ return result
+}
+
+func (pq *processQueue) isLockExpired() bool {
+ return time.Now().Sub(pq.LastLockTime()) > _RebalanceLockMaxTime
+}
+
+func (pq *processQueue) isPullExpired() bool {
+ return time.Now().Sub(pq.lastPullTime) > _PullMaxIdleTime
+}
+
+func (pq *processQueue) cleanExpiredMsg(consumer defaultConsumer) {
+ if consumer.option.ConsumeOrderly {
+ return
+ }
+ var loop = 16
+ if pq.msgCache.Size() < 16 {
+ loop = pq.msgCache.Size()
+ }
+
+ for i := 0; i < loop; i++ {
+ pq.mutex.RLock()
+ if pq.msgCache.Empty() {
+ pq.mutex.RLock()
+ return
+ }
+ _, firstValue := pq.msgCache.Min()
+ msg := firstValue.(*primitive.MessageExt)
+ startTime := msg.GetProperty(primitive.PropertyConsumeStartTime)
+ if startTime != "" {
+ st, err := strconv.ParseInt(startTime, 10, 64)
+ if err != nil {
+ rlog.Warning("parse message start consume time error", map[string]interface{}{
+ "time": startTime,
+ rlog.LogKeyUnderlayError: err,
+ })
+ continue
+ }
+ if time.Now().Unix()-st <= int64(consumer.option.ConsumeTimeout) {
+ pq.mutex.RLock()
+ return
+ }
+ }
+ pq.mutex.RLock()
+
+ err := consumer.sendBack(msg, 3)
+ if err != nil {
+ rlog.Error("send message back to broker error when clean expired messages", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ continue
+ }
+ pq.removeMessage(msg)
+ }
+}
+
+func (pq *processQueue) getMaxSpan() int {
+ pq.mutex.RLock()
+ defer pq.mutex.RUnlock()
+ if pq.msgCache.Size() == 0 {
+ return 0
+ }
+ firstKey, _ := pq.msgCache.Min()
+ lastKey, _ := pq.msgCache.Max()
+ return int(lastKey.(int64) - firstKey.(int64))
+}
+
+func (pq *processQueue) getMessages() []*primitive.MessageExt {
+ return <-pq.msgCh
+}
+
+func (pq *processQueue) takeMessages(number int) []*primitive.MessageExt {
+ for pq.msgCache.Empty() {
+ time.Sleep(10 * time.Millisecond)
+ }
+ result := make([]*primitive.MessageExt, number)
+ i := 0
+ pq.mutex.Lock()
+ for ; i < number; i++ {
+ k, v := pq.msgCache.Min()
+ if v == nil {
+ break
+ }
+ result[i] = v.(*primitive.MessageExt)
+ pq.consumingMsgOrderlyTreeMap.Put(k, v)
+ pq.msgCache.Remove(k)
+ }
+ pq.mutex.Unlock()
+ return result[:i]
+}
+
+func (pq *processQueue) Min() int64 {
+ if pq.msgCache.Empty() {
+ return -1
+ }
+ k, _ := pq.msgCache.Min()
+ if k != nil {
+ return k.(int64)
+ }
+ return -1
+}
+
+func (pq *processQueue) Max() int64 {
+ if pq.msgCache.Empty() {
+ return -1
+ }
+ k, _ := pq.msgCache.Max()
+ if k != nil {
+ return k.(int64)
+ }
+ return -1
+}
+
+func (pq *processQueue) MinOrderlyCache() int64 {
+ if pq.consumingMsgOrderlyTreeMap.Empty() {
+ return -1
+ }
+ k, _ := pq.consumingMsgOrderlyTreeMap.Min()
+ if k != nil {
+ return k.(int64)
+ }
+ return -1
+}
+
+func (pq *processQueue) MaxOrderlyCache() int64 {
+ if pq.consumingMsgOrderlyTreeMap.Empty() {
+ return -1
+ }
+ k, _ := pq.consumingMsgOrderlyTreeMap.Max()
+ if k != nil {
+ return k.(int64)
+ }
+ return -1
+}
+
+func (pq *processQueue) clear() {
+ pq.mutex.Lock()
+ pq.msgCache.Clear()
+ pq.cachedMsgCount = 0
+ pq.cachedMsgSize = 0
+ pq.queueOffsetMax = 0
+}
+
+func (pq *processQueue) commit() int64 {
+ pq.mutex.Lock()
+ defer pq.mutex.Unlock()
+
+ var offset int64
+ iter, _ := pq.consumingMsgOrderlyTreeMap.Max()
+ if iter != nil {
+ offset = iter.(int64)
+ }
+ pq.cachedMsgCount -= int64(pq.consumingMsgOrderlyTreeMap.Size())
+ pq.consumingMsgOrderlyTreeMap.Each(func(key interface{}, value interface{}) {
+ msg := value.(*primitive.MessageExt)
+ pq.cachedMsgSize -= int64(len(msg.Body))
+ })
+ pq.consumingMsgOrderlyTreeMap.Clear()
+ return offset + 1
+}
+
+func (pq *processQueue) currentInfo() internal.ProcessQueueInfo {
+ pq.mutex.RLock()
+ defer pq.mutex.RUnlock()
+ info := internal.ProcessQueueInfo{
+ Locked: pq.locked.Load(),
+ TryUnlockTimes: pq.tryUnlockTimes,
+ LastLockTimestamp: pq.LastLockTime().UnixNano() / int64(time.Millisecond),
+ Dropped: pq.dropped.Load(),
+ LastPullTimestamp: pq.lastPullTime.UnixNano() / int64(time.Millisecond),
+ LastConsumeTimestamp: pq.LastConsumeTime().UnixNano() / int64(time.Millisecond),
+ }
+
+ if !pq.msgCache.Empty() {
+ info.CachedMsgMinOffset = pq.Min()
+ info.CachedMsgMaxOffset = pq.Max()
+ info.CachedMsgCount = pq.msgCache.Size()
+ info.CachedMsgSizeInMiB = pq.cachedMsgSize / int64(1024*1024)
+ }
+
+ if !pq.consumingMsgOrderlyTreeMap.Empty() {
+ info.TransactionMsgMinOffset = pq.MinOrderlyCache()
+ info.TransactionMsgMaxOffset = pq.MaxOrderlyCache()
+ info.TransactionMsgCount = pq.consumingMsgOrderlyTreeMap.Size()
+ }
+
+ return info
+}
diff --git a/consumer/pull_consumer.go b/consumer/pull_consumer.go
new file mode 100644
index 0000000..0523f08
--- /dev/null
+++ b/consumer/pull_consumer.go
@@ -0,0 +1,253 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "context"
+ "fmt"
+ "sync"
+ "sync/atomic"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+type PullConsumer interface {
+ // Start
+ Start()
+
+ // Shutdown refuse all new pull operation, finish all submitted.
+ Shutdown()
+
+ // Pull pull message of topic, selector indicate which queue to pull.
+ Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error)
+
+ // PullFrom pull messages of queue from the offset to offset + numbers
+ PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error)
+
+ // updateOffset update offset of queue in mem
+ UpdateOffset(queue *primitive.MessageQueue, offset int64) error
+
+ // PersistOffset persist all offset in mem.
+ PersistOffset(ctx context.Context) error
+
+ // CurrentOffset return the current offset of queue in mem.
+ CurrentOffset(queue *primitive.MessageQueue) (int64, error)
+}
+
+var (
+ queueCounterTable sync.Map
+)
+
+type defaultPullConsumer struct {
+ *defaultConsumer
+
+ option consumerOptions
+ client internal.RMQClient
+ GroupName string
+ Model MessageModel
+ UnitMode bool
+
+ interceptor primitive.Interceptor
+}
+
+func NewPullConsumer(options ...Option) (*defaultPullConsumer, error) {
+ defaultOpts := defaultPullConsumerOptions()
+ for _, apply := range options {
+ apply(&defaultOpts)
+ }
+
+ srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
+ if err != nil {
+ return nil, errors.Wrap(err, "new Namesrv failed.")
+ }
+
+ dc := &defaultConsumer{
+ client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
+ consumerGroup: defaultOpts.GroupName,
+ cType: _PullConsume,
+ state: int32(internal.StateCreateJust),
+ prCh: make(chan PullRequest, 4),
+ model: defaultOpts.ConsumerModel,
+ option: defaultOpts,
+
+ namesrv: srvs,
+ }
+
+ c := &defaultPullConsumer{
+ defaultConsumer: dc,
+ }
+ return c, nil
+}
+
+func (c *defaultPullConsumer) Start() error {
+ atomic.StoreInt32(&c.state, int32(internal.StateRunning))
+
+ var err error
+ c.once.Do(func() {
+ err = c.start()
+ if err != nil {
+ return
+ }
+ })
+
+ return err
+}
+
+func (c *defaultPullConsumer) Pull(ctx context.Context, topic string, selector MessageSelector, numbers int) (*primitive.PullResult, error) {
+ mq := c.getNextQueueOf(topic)
+ if mq == nil {
+ return nil, fmt.Errorf("prepard to pull topic: %s, but no queue is founded", topic)
+ }
+
+ data := buildSubscriptionData(mq.Topic, selector)
+ result, err := c.pull(context.Background(), mq, data, c.nextOffsetOf(mq), numbers)
+
+ if err != nil {
+ return nil, err
+ }
+
+ c.processPullResult(mq, result, data)
+ return result, nil
+}
+
+func (c *defaultPullConsumer) getNextQueueOf(topic string) *primitive.MessageQueue {
+ queues, err := c.defaultConsumer.namesrv.FetchSubscribeMessageQueues(topic)
+ if err != nil && len(queues) > 0 {
+ rlog.Error("get next mq error", map[string]interface{}{
+ rlog.LogKeyTopic: topic,
+ rlog.LogKeyUnderlayError: err.Error(),
+ })
+ return nil
+ }
+ var index int64
+ v, exist := queueCounterTable.Load(topic)
+ if !exist {
+ index = -1
+ queueCounterTable.Store(topic, 0)
+ } else {
+ index = v.(int64)
+ }
+
+ return queues[int(atomic.AddInt64(&index, 1))%len(queues)]
+}
+
+// SubscribeWithChan ack manually
+func (c *defaultPullConsumer) SubscribeWithChan(topic, selector MessageSelector) (chan *primitive.Message, error) {
+ return nil, nil
+}
+
+// SubscribeWithFunc ack automatic
+func (c *defaultPullConsumer) SubscribeWithFunc(topic, selector MessageSelector,
+ f func(msg *primitive.Message) ConsumeResult) error {
+ return nil
+}
+
+func (c *defaultPullConsumer) ACK(msg *primitive.Message, result ConsumeResult) {
+
+}
+
+func (dc *defaultConsumer) checkPull(ctx context.Context, mq *primitive.MessageQueue, offset int64, numbers int) error {
+ err := dc.makeSureStateOK()
+ if err != nil {
+ return err
+ }
+
+ if mq == nil {
+ return utils.ErrMQEmpty
+ }
+
+ if offset < 0 {
+ return utils.ErrOffset
+ }
+
+ if numbers <= 0 {
+ return utils.ErrNumbers
+ }
+ return nil
+}
+
+// TODO: add timeout limit
+// TODO: add hook
+func (c *defaultPullConsumer) pull(ctx context.Context, mq *primitive.MessageQueue, data *internal.SubscriptionData,
+ offset int64, numbers int) (*primitive.PullResult, error) {
+
+ if err := c.checkPull(ctx, mq, offset, numbers); err != nil {
+ return nil, err
+ }
+
+ c.subscriptionAutomatically(mq.Topic)
+
+ sysFlag := buildSysFlag(false, true, true, false)
+
+ pullResp, err := c.pullInner(ctx, mq, data, offset, numbers, sysFlag, 0)
+ if err != nil {
+ return nil, err
+ }
+ c.processPullResult(mq, pullResp, data)
+
+ return pullResp, err
+}
+
+func (c *defaultPullConsumer) makeSureStateOK() error {
+ if atomic.LoadInt32(&c.state) != int32(internal.StateRunning) {
+ return fmt.Errorf("the consumer state is [%d], not running", c.state)
+ }
+ return nil
+}
+
+func (c *defaultPullConsumer) nextOffsetOf(queue *primitive.MessageQueue) int64 {
+ return c.computePullFromWhere(queue)
+}
+
+// PullFrom pull messages of queue from the offset to offset + numbers
+func (c *defaultPullConsumer) PullFrom(ctx context.Context, queue *primitive.MessageQueue, offset int64, numbers int) (*primitive.PullResult, error) {
+ if err := c.checkPull(ctx, queue, offset, numbers); err != nil {
+ return nil, err
+ }
+
+ selector := MessageSelector{}
+ data := buildSubscriptionData(queue.Topic, selector)
+
+ return c.pull(ctx, queue, data, offset, numbers)
+}
+
+// updateOffset update offset of queue in mem
+func (c *defaultPullConsumer) UpdateOffset(queue *primitive.MessageQueue, offset int64) error {
+ return c.updateOffset(queue, offset)
+}
+
+// PersistOffset persist all offset in mem.
+func (c *defaultPullConsumer) PersistOffset(ctx context.Context) error {
+ return c.persistConsumerOffset()
+}
+
+// CurrentOffset return the current offset of queue in mem.
+func (c *defaultPullConsumer) CurrentOffset(queue *primitive.MessageQueue) (int64, error) {
+ v := c.queryOffset(queue)
+ return v, nil
+}
+
+// Shutdown close defaultConsumer, refuse new request.
+func (c *defaultPullConsumer) Shutdown() error {
+ return c.defaultConsumer.shutdown()
+}
diff --git a/consumer/pull_consumer_test.go b/consumer/pull_consumer_test.go
new file mode 100644
index 0000000..f7bc454
--- /dev/null
+++ b/consumer/pull_consumer_test.go
@@ -0,0 +1,18 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
diff --git a/consumer/push_consumer.go b/consumer/push_consumer.go
new file mode 100644
index 0000000..d2aee5b
--- /dev/null
+++ b/consumer/push_consumer.go
@@ -0,0 +1,1183 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "context"
+ "fmt"
+ "math"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+// In most scenarios, this is the mostly recommended usage to consume messages.
+//
+// Technically speaking, this push client is virtually a wrapper of the underlying pull service. Specifically, on
+// arrival of messages pulled from brokers, it roughly invokes the registered callback handler to feed the messages.
+//
+// See quick start/Consumer in the example module for a typical usage.
+//
+// <strong>Thread Safety:</strong> After initialization, the instance can be regarded as thread-safe.
+
+const (
+ Mb = 1024 * 1024
+)
+
+type PushConsumerCallback struct {
+ topic string
+ f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)
+}
+
+func (callback PushConsumerCallback) UniqueID() string {
+ return callback.topic
+}
+
+type pushConsumer struct {
+ *defaultConsumer
+ queueFlowControlTimes int
+ queueMaxSpanFlowControlTimes int
+ consumeFunc utils.Set
+ submitToConsume func(*processQueue, *primitive.MessageQueue)
+ subscribedTopic map[string]string
+ interceptor primitive.Interceptor
+ queueLock *QueueLock
+ done chan struct{}
+ closeOnce sync.Once
+}
+
+func NewPushConsumer(opts ...Option) (*pushConsumer, error) {
+ defaultOpts := defaultPushConsumerOptions()
+ for _, apply := range opts {
+ apply(&defaultOpts)
+ }
+ srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
+ if err != nil {
+ return nil, errors.Wrap(err, "new Namesrv failed.")
+ }
+ if !defaultOpts.Credentials.IsEmpty() {
+ srvs.SetCredentials(defaultOpts.Credentials)
+ }
+ defaultOpts.Namesrv = srvs
+
+ if defaultOpts.Namespace != "" {
+ defaultOpts.GroupName = defaultOpts.Namespace + "%" + defaultOpts.GroupName
+ }
+
+ dc := &defaultConsumer{
+ client: internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, nil),
+ consumerGroup: defaultOpts.GroupName,
+ cType: _PushConsume,
+ state: int32(internal.StateCreateJust),
+ prCh: make(chan PullRequest, 4),
+ model: defaultOpts.ConsumerModel,
+ consumeOrderly: defaultOpts.ConsumeOrderly,
+ fromWhere: defaultOpts.FromWhere,
+ allocate: defaultOpts.Strategy,
+ option: defaultOpts,
+ namesrv: srvs,
+ }
+
+ p := &pushConsumer{
+ defaultConsumer: dc,
+ subscribedTopic: make(map[string]string, 0),
+ queueLock: newQueueLock(),
+ done: make(chan struct{}, 1),
+ consumeFunc: utils.NewSet(),
+ }
+ dc.mqChanged = p.messageQueueChanged
+ if p.consumeOrderly {
+ p.submitToConsume = p.consumeMessageOrderly
+ } else {
+ p.submitToConsume = p.consumeMessageCurrently
+ }
+
+ p.interceptor = primitive.ChainInterceptors(p.option.Interceptors...)
+
+ return p, nil
+}
+
+func (pc *pushConsumer) Start() error {
+ var err error
+ pc.once.Do(func() {
+ rlog.Info("the consumer start beginning", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ "messageModel": pc.model,
+ "unitMode": pc.unitMode,
+ })
+ atomic.StoreInt32(&pc.state, int32(internal.StateStartFailed))
+ pc.validate()
+
+ err = pc.client.RegisterConsumer(pc.consumerGroup, pc)
+ if err != nil {
+ rlog.Error("the consumer group has been created, specify another one", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ })
+ err = ErrCreated
+ return
+ }
+
+ err = pc.defaultConsumer.start()
+ if err != nil {
+ return
+ }
+
+ go func() {
+ // todo start clean msg expired
+ for {
+ select {
+ case pr := <-pc.prCh:
+ go func() {
+ pc.pullMessage(&pr)
+ }()
+ case <-pc.done:
+ rlog.Info("push consumer close pullConsumer listener.", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ })
+ return
+ }
+ }
+ }()
+
+ go primitive.WithRecover(func() {
+ // initial lock.
+ if !pc.consumeOrderly {
+ return
+ }
+
+ time.Sleep(1000 * time.Millisecond)
+ pc.lockAll()
+
+ lockTicker := time.NewTicker(pc.option.RebalanceLockInterval)
+ defer lockTicker.Stop()
+ for {
+ select {
+ case <-lockTicker.C:
+ pc.lockAll()
+ case <-pc.done:
+ rlog.Info("push consumer close tick.", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ })
+ return
+ }
+ }
+ })
+ })
+
+ if err != nil {
+ return err
+ }
+
+ pc.client.UpdateTopicRouteInfo()
+ for k := range pc.subscribedTopic {
+ _, exist := pc.topicSubscribeInfoTable.Load(k)
+ if !exist {
+ pc.client.Shutdown()
+ return fmt.Errorf("the topic=%s route info not found, it may not exist", k)
+ }
+ }
+ pc.client.CheckClientInBroker()
+ pc.client.SendHeartbeatToAllBrokerWithLock()
+ pc.client.RebalanceImmediately()
+
+ return err
+}
+
+func (pc *pushConsumer) Shutdown() error {
+ var err error
+ pc.closeOnce.Do(func() {
+ close(pc.done)
+
+ pc.client.UnregisterConsumer(pc.consumerGroup)
+ err = pc.defaultConsumer.shutdown()
+ })
+
+ return err
+}
+
+func (pc *pushConsumer) Subscribe(topic string, selector MessageSelector,
+ f func(context.Context, ...*primitive.MessageExt) (ConsumeResult, error)) error {
+ if atomic.LoadInt32(&pc.state) != int32(internal.StateCreateJust) {
+ return errors.New("subscribe topic only started before")
+ }
+ if pc.option.Namespace != "" {
+ topic = pc.option.Namespace + "%" + topic
+ }
+ data := buildSubscriptionData(topic, selector)
+ pc.subscriptionDataTable.Store(topic, data)
+ pc.subscribedTopic[topic] = ""
+
+ pc.consumeFunc.Add(&PushConsumerCallback{
+ f: f,
+ topic: topic,
+ })
+ return nil
+}
+
+func (pc *pushConsumer) Unsubscribe(string) error {
+ return nil
+}
+
+func (pc *pushConsumer) Rebalance() {
+ pc.defaultConsumer.doBalance()
+}
+
+func (pc *pushConsumer) PersistConsumerOffset() error {
+ return pc.defaultConsumer.persistConsumerOffset()
+}
+
+func (pc *pushConsumer) UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
+ pc.defaultConsumer.updateTopicSubscribeInfo(topic, mqs)
+}
+
+func (pc *pushConsumer) IsSubscribeTopicNeedUpdate(topic string) bool {
+ return pc.defaultConsumer.isSubscribeTopicNeedUpdate(topic)
+}
+
+func (pc *pushConsumer) SubscriptionDataList() []*internal.SubscriptionData {
+ return pc.defaultConsumer.SubscriptionDataList()
+}
+
+func (pc *pushConsumer) IsUnitMode() bool {
+ return pc.unitMode
+}
+
+func (pc *pushConsumer) GetcType() string {
+ return string(pc.cType)
+}
+
+func (pc *pushConsumer) GetModel() string {
+ return pc.model.String()
+}
+
+func (pc *pushConsumer) GetWhere() string {
+ switch pc.fromWhere {
+ case ConsumeFromLastOffset:
+ return "CONSUME_FROM_LAST_OFFSET"
+ case ConsumeFromFirstOffset:
+ return "CONSUME_FROM_FIRST_OFFSET"
+ case ConsumeFromTimestamp:
+ return "CONSUME_FROM_TIMESTAMP"
+ default:
+ return "UNKOWN"
+ }
+
+}
+
+func (pc *pushConsumer) GetConsumerRunningInfo() *internal.ConsumerRunningInfo {
+ info := internal.NewConsumerRunningInfo()
+
+ pc.subscriptionDataTable.Range(func(key, value interface{}) bool {
+ topic := key.(string)
+ info.SubscriptionData[value.(*internal.SubscriptionData)] = true
+ status := internal.ConsumeStatus{
+ PullRT: getPullRT(topic, pc.consumerGroup).avgpt,
+ PullTPS: getPullTPS(topic, pc.consumerGroup).tps,
+ ConsumeRT: getConsumeRT(topic, pc.consumerGroup).avgpt,
+ ConsumeOKTPS: getConsumeOKTPS(topic, pc.consumerGroup).tps,
+ ConsumeFailedTPS: getConsumeFailedTPS(topic, pc.consumerGroup).tps,
+ ConsumeFailedMsgs: topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + pc.consumerGroup).sum,
+ }
+ info.StatusTable[topic] = status
+ return true
+ })
+
+ pc.processQueueTable.Range(func(key, value interface{}) bool {
+ mq := key.(primitive.MessageQueue)
+ pq := value.(*processQueue)
+ pInfo := pq.currentInfo()
+ pInfo.CommitOffset = pc.storage.read(&mq, _ReadMemoryThenStore)
+ info.MQTable[mq] = pInfo
+ return true
+ })
+
+ nsAddr := ""
+ for _, value := range pc.namesrv.AddrList() {
+ nsAddr += fmt.Sprintf("%s;", value)
+ }
+ info.Properties[internal.PropNameServerAddr] = nsAddr
+ info.Properties[internal.PropConsumeType] = string(pc.cType)
+ info.Properties[internal.PropConsumeOrderly] = strconv.FormatBool(pc.consumeOrderly)
+ info.Properties[internal.PropThreadPoolCoreSize] = "-1"
+ info.Properties[internal.PropConsumerStartTimestamp] = strconv.FormatInt(pc.consumerStartTimestamp, 10)
+ return info
+}
+
+func (pc *pushConsumer) messageQueueChanged(topic string, mqAll, mqDivided []*primitive.MessageQueue) {
+ v, exit := pc.subscriptionDataTable.Load(topic)
+ if !exit {
+ return
+ }
+ data := v.(*internal.SubscriptionData)
+ newVersion := time.Now().UnixNano()
+ rlog.Info("the MessageQueue changed, version also updated", map[string]interface{}{
+ rlog.LogKeyValueChangedFrom: data.SubVersion,
+ rlog.LogKeyValueChangedTo: newVersion,
+ })
+ data.SubVersion = newVersion
+
+ // TODO: optimize
+ count := 0
+ pc.processQueueTable.Range(func(key, value interface{}) bool {
+ count++
+ return true
+ })
+ if count > 0 {
+ if pc.option.PullThresholdForTopic != -1 {
+ newVal := pc.option.PullThresholdForTopic / count
+ if newVal == 0 {
+ newVal = 1
+ }
+ rlog.Info("The PullThresholdForTopic is changed", map[string]interface{}{
+ rlog.LogKeyValueChangedFrom: pc.option.PullThresholdForTopic,
+ rlog.LogKeyValueChangedTo: newVal,
+ })
+ pc.option.PullThresholdForTopic = newVal
+ }
+
+ if pc.option.PullThresholdSizeForTopic != -1 {
+ newVal := pc.option.PullThresholdSizeForTopic / count
+ if newVal == 0 {
+ newVal = 1
+ }
+ rlog.Info("The PullThresholdSizeForTopic is changed", map[string]interface{}{
+ rlog.LogKeyValueChangedFrom: pc.option.PullThresholdSizeForTopic,
+ rlog.LogKeyValueChangedTo: newVal,
+ })
+ }
+ }
+ pc.client.SendHeartbeatToAllBrokerWithLock()
+}
+
+func (pc *pushConsumer) validate() {
+ internal.ValidateGroup(pc.consumerGroup)
+
+ if pc.consumerGroup == internal.DefaultConsumerGroup {
+ // TODO FQA
+ rlog.Error(fmt.Sprintf("consumerGroup can't equal [%s], please specify another one.", internal.DefaultConsumerGroup), nil)
+ }
+
+ if len(pc.subscribedTopic) == 0 {
+ rlog.Error("number of subscribed topics is 0.", nil)
+ }
+
+ if pc.option.ConsumeConcurrentlyMaxSpan < 1 || pc.option.ConsumeConcurrentlyMaxSpan > 65535 {
+ if pc.option.ConsumeConcurrentlyMaxSpan == 0 {
+ pc.option.ConsumeConcurrentlyMaxSpan = 1000
+ } else {
+ rlog.Error("option.ConsumeConcurrentlyMaxSpan out of range [1, 65535]", nil)
+ }
+ }
+
+ if pc.option.PullThresholdForQueue < 1 || pc.option.PullThresholdForQueue > 65535 {
+ if pc.option.PullThresholdForQueue == 0 {
+ pc.option.PullThresholdForQueue = 1024
+ } else {
+ rlog.Error("option.PullThresholdForQueue out of range [1, 65535]", nil)
+ }
+ }
+
+ if pc.option.PullThresholdForTopic < 1 || pc.option.PullThresholdForTopic > 6553500 {
+ if pc.option.PullThresholdForTopic == 0 {
+ pc.option.PullThresholdForTopic = 102400
+ } else {
+ rlog.Error("option.PullThresholdForTopic out of range [1, 6553500]", nil)
+ }
+ }
+
+ if pc.option.PullThresholdSizeForQueue < 1 || pc.option.PullThresholdSizeForQueue > 1024 {
+ if pc.option.PullThresholdSizeForQueue == 0 {
+ pc.option.PullThresholdSizeForQueue = 512
+ } else {
+ rlog.Error("option.PullThresholdSizeForQueue out of range [1, 1024]", nil)
+ }
+ }
+
+ if pc.option.PullThresholdSizeForTopic < 1 || pc.option.PullThresholdSizeForTopic > 102400 {
+ if pc.option.PullThresholdSizeForTopic == 0 {
+ pc.option.PullThresholdSizeForTopic = 51200
+ } else {
+ rlog.Error("option.PullThresholdSizeForTopic out of range [1, 102400]", nil)
+ }
+ }
+
+ if pc.option.PullInterval < 0 || pc.option.PullInterval > 65535 {
+ rlog.Error("option.PullInterval out of range [0, 65535]", nil)
+ }
+
+ if pc.option.ConsumeMessageBatchMaxSize < 1 || pc.option.ConsumeMessageBatchMaxSize > 1024 {
+ if pc.option.ConsumeMessageBatchMaxSize == 0 {
+ pc.option.ConsumeMessageBatchMaxSize = 512
+ } else {
+ rlog.Error("option.ConsumeMessageBatchMaxSize out of range [1, 1024]", nil)
+ }
+ }
+
+ if pc.option.PullBatchSize < 1 || pc.option.PullBatchSize > 1024 {
+ if pc.option.PullBatchSize == 0 {
+ pc.option.PullBatchSize = 32
+ } else {
+ rlog.Error("option.PullBatchSize out of range [1, 1024]", nil)
+ }
+ }
+}
+
+func (pc *pushConsumer) pullMessage(request *PullRequest) {
+ rlog.Debug("start a new Pull Message task for PullRequest", map[string]interface{}{
+ rlog.LogKeyPullRequest: request.String(),
+ })
+ var sleepTime time.Duration
+ pq := request.pq
+ go primitive.WithRecover(func() {
+ for {
+ select {
+ case <-pc.done:
+ rlog.Info("push consumer close pullMessage.", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ })
+ return
+ default:
+ pc.submitToConsume(request.pq, request.mq)
+ }
+ }
+ })
+
+ for {
+ NEXT:
+ select {
+ case <-pc.done:
+ rlog.Info("push consumer close message handle.", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ })
+ return
+ default:
+ }
+
+ if pq.IsDroppd() {
+ rlog.Debug("the request was dropped, so stop task", map[string]interface{}{
+ rlog.LogKeyPullRequest: request.String(),
+ })
+ return
+ }
+ if sleepTime > 0 {
+ rlog.Debug(fmt.Sprintf("pull MessageQueue: %d sleep %d ms for mq: %v", request.mq.QueueId, sleepTime/time.Millisecond, request.mq), nil)
+ time.Sleep(sleepTime)
+ }
+ // reset time
+ sleepTime = pc.option.PullInterval
+ pq.lastPullTime = time.Now()
+ err := pc.makeSureStateOK()
+ if err != nil {
+ rlog.Warning("consumer state error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err.Error(),
+ })
+ sleepTime = _PullDelayTimeWhenError
+ goto NEXT
+ }
+
+ if pc.pause {
+ rlog.Debug(fmt.Sprintf("consumer [%s] of [%s] was paused, execute pull request [%s] later",
+ pc.option.InstanceName, pc.consumerGroup, request.String()), nil)
+ sleepTime = _PullDelayTimeWhenSuspend
+ goto NEXT
+ }
+
+ cachedMessageSizeInMiB := int(pq.cachedMsgSize / Mb)
+ if pq.cachedMsgCount > pc.option.PullThresholdForQueue {
+ if pc.queueFlowControlTimes%1000 == 0 {
+ rlog.Warning("the cached message count exceeds the threshold, so do flow control", map[string]interface{}{
+ "PullThresholdForQueue": pc.option.PullThresholdForQueue,
+ "minOffset": pq.Min(),
+ "maxOffset": pq.Max(),
+ "count": pq.msgCache,
+ "size(MiB)": cachedMessageSizeInMiB,
+ "flowControlTimes": pc.queueFlowControlTimes,
+ rlog.LogKeyPullRequest: request.String(),
+ })
+ }
+ pc.queueFlowControlTimes++
+ sleepTime = _PullDelayTimeWhenFlowControl
+ goto NEXT
+ }
+
+ if cachedMessageSizeInMiB > pc.option.PullThresholdSizeForQueue {
+ if pc.queueFlowControlTimes%1000 == 0 {
+ rlog.Warning("the cached message size exceeds the threshold, so do flow control", map[string]interface{}{
+ "PullThresholdSizeForQueue": pc.option.PullThresholdSizeForQueue,
+ "minOffset": pq.Min(),
+ "maxOffset": pq.Max(),
+ "count": pq.msgCache,
+ "size(MiB)": cachedMessageSizeInMiB,
+ "flowControlTimes": pc.queueFlowControlTimes,
+ rlog.LogKeyPullRequest: request.String(),
+ })
+ }
+ pc.queueFlowControlTimes++
+ sleepTime = _PullDelayTimeWhenFlowControl
+ goto NEXT
+ }
+
+ if !pc.consumeOrderly {
+ if pq.getMaxSpan() > pc.option.ConsumeConcurrentlyMaxSpan {
+ if pc.queueMaxSpanFlowControlTimes%1000 == 0 {
+ rlog.Warning("the queue's messages span too long, so do flow control", map[string]interface{}{
+ "ConsumeConcurrentlyMaxSpan": pc.option.ConsumeConcurrentlyMaxSpan,
+ "minOffset": pq.Min(),
+ "maxOffset": pq.Max(),
+ "maxSpan": pq.getMaxSpan(),
+ "flowControlTimes": pc.queueFlowControlTimes,
+ rlog.LogKeyPullRequest: request.String(),
+ })
+ }
+ sleepTime = _PullDelayTimeWhenFlowControl
+ goto NEXT
+ }
+ } else {
+ if pq.IsLock() {
+ if !request.lockedFirst {
+ offset := pc.computePullFromWhere(request.mq)
+ brokerBusy := offset < request.nextOffset
+ rlog.Info("the first time to pull message, so fix offset from broker, offset maybe changed", map[string]interface{}{
+ rlog.LogKeyPullRequest: request.String(),
+ rlog.LogKeyValueChangedFrom: request.nextOffset,
+ rlog.LogKeyValueChangedTo: offset,
+ "brokerBusy": brokerBusy,
+ })
+ if brokerBusy {
+ rlog.Info("[NOTIFY_ME] the first time to pull message, but pull request offset larger than "+
+ "broker consume offset", map[string]interface{}{"offset": offset})
+ }
+ request.lockedFirst = true
+ request.nextOffset = offset
+ }
+ } else {
+ rlog.Info("pull message later because not locked in broker", map[string]interface{}{
+ rlog.LogKeyPullRequest: request.String(),
+ })
+ sleepTime = _PullDelayTimeWhenError
+ goto NEXT
+ }
+ }
+
+ v, exist := pc.subscriptionDataTable.Load(request.mq.Topic)
+ if !exist {
+ rlog.Info("find the consumer's subscription failed", map[string]interface{}{
+ rlog.LogKeyPullRequest: request.String(),
+ })
+ sleepTime = _PullDelayTimeWhenError
+ goto NEXT
+ }
+ beginTime := time.Now()
+ var (
+ commitOffsetEnable bool
+ commitOffsetValue int64
+ subExpression string
+ )
+
+ if pc.model == Clustering {
+ commitOffsetValue = pc.storage.read(request.mq, _ReadFromMemory)
+ if commitOffsetValue > 0 {
+ commitOffsetEnable = true
+ }
+ }
+
+ sd := v.(*internal.SubscriptionData)
+ classFilter := sd.ClassFilterMode
+ if pc.option.PostSubscriptionWhenPull && classFilter {
+ subExpression = sd.SubString
+ }
+
+ sysFlag := buildSysFlag(commitOffsetEnable, true, subExpression != "", classFilter)
+
+ pullRequest := &internal.PullMessageRequestHeader{
+ ConsumerGroup: pc.consumerGroup,
+ Topic: request.mq.Topic,
+ QueueId: int32(request.mq.QueueId),
+ QueueOffset: request.nextOffset,
+ MaxMsgNums: pc.option.PullBatchSize,
+ SysFlag: sysFlag,
+ CommitOffset: commitOffsetValue,
+ SubExpression: _SubAll,
+ ExpressionType: string(TAG),
+ SuspendTimeoutMillis: 20 * time.Second,
+ }
+ //
+ //if data.ExpType == string(TAG) {
+ // pullRequest.SubVersion = 0
+ //} else {
+ // pullRequest.SubVersion = data.SubVersion
+ //}
+
+ brokerResult := pc.defaultConsumer.tryFindBroker(request.mq)
+ if brokerResult == nil {
+ rlog.Warning("no broker found for mq", map[string]interface{}{
+ rlog.LogKeyPullRequest: request.mq.String(),
+ })
+ sleepTime = _PullDelayTimeWhenError
+ goto NEXT
+ }
+
+ if brokerResult.Slave {
+ pullRequest.SysFlag = clearCommitOffsetFlag(pullRequest.SysFlag)
+ }
+
+ result, err := pc.client.PullMessage(context.Background(), brokerResult.BrokerAddr, pullRequest)
+ if err != nil {
+ rlog.Warning("pull message from broker error", map[string]interface{}{
+ rlog.LogKeyBroker: brokerResult.BrokerAddr,
+ rlog.LogKeyUnderlayError: err.Error(),
+ })
+ sleepTime = _PullDelayTimeWhenError
+ goto NEXT
+ }
+
+ if result.Status == primitive.PullBrokerTimeout {
+ rlog.Warning("pull broker timeout", map[string]interface{}{
+ rlog.LogKeyBroker: brokerResult.BrokerAddr,
+ })
+ sleepTime = _PullDelayTimeWhenError
+ goto NEXT
+ }
+
+ switch result.Status {
+ case primitive.PullFound:
+ rlog.Debug(fmt.Sprintf("Topic: %s, QueueId: %d found messages.", request.mq.Topic, request.mq.QueueId), nil)
+ prevRequestOffset := request.nextOffset
+ request.nextOffset = result.NextBeginOffset
+
+ rt := time.Now().Sub(beginTime) / time.Millisecond
+ increasePullRT(pc.consumerGroup, request.mq.Topic, int64(rt))
+
+ pc.processPullResult(request.mq, result, sd)
+
+ msgFounded := result.GetMessageExts()
+ firstMsgOffset := int64(math.MaxInt64)
+ if msgFounded != nil && len(msgFounded) != 0 {
+ firstMsgOffset = msgFounded[0].QueueOffset
+ increasePullTPS(pc.consumerGroup, request.mq.Topic, len(msgFounded))
+ pq.putMessage(msgFounded...)
+ }
+ if result.NextBeginOffset < prevRequestOffset || firstMsgOffset < prevRequestOffset {
+ rlog.Warning("[BUG] pull message result maybe data wrong", map[string]interface{}{
+ "nextBeginOffset": result.NextBeginOffset,
+ "firstMsgOffset": firstMsgOffset,
+ "prevRequestOffset": prevRequestOffset,
+ })
+ }
+ case primitive.PullNoNewMsg:
+ rlog.Debug(fmt.Sprintf("Topic: %s, QueueId: %d no more msg, current offset: %d, next offset: %d",
+ request.mq.Topic, request.mq.QueueId, pullRequest.QueueOffset, result.NextBeginOffset), nil)
+ case primitive.PullNoMsgMatched:
+ request.nextOffset = result.NextBeginOffset
+ pc.correctTagsOffset(request)
+ case primitive.PullOffsetIllegal:
+ rlog.Warning("the pull request offset illegal", map[string]interface{}{
+ rlog.LogKeyPullRequest: request.String(),
+ "result": result.String(),
+ })
+ request.nextOffset = result.NextBeginOffset
+ pq.WithDropped(true)
+ time.Sleep(10 * time.Second)
+ pc.storage.update(request.mq, request.nextOffset, false)
+ pc.storage.persist([]*primitive.MessageQueue{request.mq})
+ pc.processQueueTable.Delete(request.mq)
+ rlog.Warning(fmt.Sprintf("fix the pull request offset: %s", request.String()), nil)
+ default:
+ rlog.Warning(fmt.Sprintf("unknown pull status: %v", result.Status), nil)
+ sleepTime = _PullDelayTimeWhenError
+ }
+ }
+}
+
+func (pc *pushConsumer) correctTagsOffset(pr *PullRequest) {
+ // TODO
+}
+
+func (pc *pushConsumer) sendMessageBack(brokerName string, msg *primitive.MessageExt, delayLevel int) bool {
+ var brokerAddr string
+ if len(brokerName) != 0 {
+ brokerAddr = pc.defaultConsumer.namesrv.FindBrokerAddrByName(brokerName)
+ } else {
+ brokerAddr = msg.StoreHost
+ }
+ _, err := pc.client.InvokeSync(context.Background(), brokerAddr, pc.buildSendBackRequest(msg, delayLevel), 3*time.Second)
+ if err != nil {
+ return false
+ }
+ return true
+}
+
+func (pc *pushConsumer) buildSendBackRequest(msg *primitive.MessageExt, delayLevel int) *remote.RemotingCommand {
+ req := &internal.ConsumerSendMsgBackRequestHeader{
+ Group: pc.consumerGroup,
+ OriginTopic: msg.Topic,
+ Offset: msg.CommitLogOffset,
+ DelayLevel: delayLevel,
+ OriginMsgId: msg.MsgId,
+ MaxReconsumeTimes: pc.getMaxReconsumeTimes(),
+ }
+
+ return remote.NewRemotingCommand(internal.ReqConsumerSendMsgBack, req, msg.Body)
+}
+
+func (pc *pushConsumer) suspend() {
+ pc.pause = true
+ rlog.Info(fmt.Sprintf("suspend consumer: %s", pc.consumerGroup), nil)
+}
+
+func (pc *pushConsumer) resume() {
+ pc.pause = false
+ pc.doBalance()
+ rlog.Info(fmt.Sprintf("resume consumer: %s", pc.consumerGroup), nil)
+}
+
+func (pc *pushConsumer) resetOffset(topic string, table map[primitive.MessageQueue]int64) {
+ //topic := cmd.ExtFields["topic"]
+ //group := cmd.ExtFields["group"]
+ //if topic == "" || group == "" {
+ // rlog.Warning("received reset offset command from: %s, but missing params.", from)
+ // return
+ //}
+ //t, err := strconv.ParseInt(cmd.ExtFields["timestamp"], 10, 64)
+ //if err != nil {
+ // rlog.Warning("received reset offset command from: %s, but parse time error: %s", err.Error())
+ // return
+ //}
+ //rlog.Infof("invoke reset offset operation from broker. brokerAddr=%s, topic=%s, group=%s, timestamp=%v",
+ // from, topic, group, t)
+ //
+ //offsetTable := make(map[MessageQueue]int64, 0)
+ //err = json.Unmarshal(cmd.Body, &offsetTable)
+ //if err != nil {
+ // rlog.Warning("received reset offset command from: %s, but parse offset table: %s", err.Error())
+ // return
+ //}
+ //v, exist := c.consumerMap.Load(group)
+ //if !exist {
+ // rlog.Infof("[reset-offset] consumer dose not exist. group=%s", group)
+ // return
+ //}
+
+ pc.processQueueTable.Range(func(key, value interface{}) bool {
+ mq := key.(primitive.MessageQueue)
+ pq := value.(*processQueue)
+ if _, ok := table[mq]; !ok {
+ pq.WithDropped(true)
+ pq.clear()
+ }
+ return true
+ })
+ time.Sleep(10 * time.Second)
+ v, exist := pc.topicSubscribeInfoTable.Load(topic)
+ if !exist {
+ return
+ }
+ queuesOfTopic := v.([]primitive.MessageQueue)
+ for _, k := range queuesOfTopic {
+ if _, ok := table[k]; ok {
+ pc.storage.update(&k, table[k], false)
+ v, exist := pc.processQueueTable.Load(k)
+ if !exist {
+ continue
+ }
+ pq := v.(*processQueue)
+ pc.removeUnnecessaryMessageQueue(&k, pq)
+ }
+ }
+}
+
+func (pc *pushConsumer) removeUnnecessaryMessageQueue(mq *primitive.MessageQueue, pq *processQueue) bool {
+ pc.defaultConsumer.removeUnnecessaryMessageQueue(mq, pq)
+ if !pc.consumeOrderly || Clustering != pc.model {
+ return true
+ }
+ // TODO orderly
+ return true
+}
+
+func (pc *pushConsumer) consumeInner(ctx context.Context, subMsgs []*primitive.MessageExt) (ConsumeResult, error) {
+ if len(subMsgs) == 0 {
+ return ConsumeRetryLater, errors.New("msg list empty")
+ }
+
+ f, exist := pc.consumeFunc.Contains(subMsgs[0].Topic)
+
+ // fix lost retry message
+ if !exist && strings.HasPrefix(subMsgs[0].Topic, internal.RetryGroupTopicPrefix) {
+ f, exist = pc.consumeFunc.Contains(subMsgs[0].GetProperty(primitive.PropertyRetryTopic))
+ }
+
+ if !exist {
+ return ConsumeRetryLater, fmt.Errorf("the consume callback missing for topic: %s", subMsgs[0].Topic)
+ }
+
+ callback, ok := f.(*PushConsumerCallback)
+ if !ok {
+ return ConsumeRetryLater, fmt.Errorf("the consume callback assert failed for topic: %s", subMsgs[0].Topic)
+ }
+ if pc.interceptor == nil {
+ return callback.f(ctx, subMsgs...)
+ } else {
+ var container ConsumeResultHolder
+ err := pc.interceptor(ctx, subMsgs, &container, func(ctx context.Context, req, reply interface{}) error {
+ msgs := req.([]*primitive.MessageExt)
+ r, e := callback.f(ctx, msgs...)
+
+ realReply := reply.(*ConsumeResultHolder)
+ realReply.ConsumeResult = r
+
+ msgCtx, _ := primitive.GetConsumerCtx(ctx)
+ msgCtx.Success = realReply.ConsumeResult == ConsumeSuccess
+ if realReply.ConsumeResult == ConsumeSuccess {
+ msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
+ } else {
+ msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
+ }
+ return e
+ })
+ return container.ConsumeResult, err
+ }
+}
+
+// resetRetryAndNamespace modify retry message.
+func (pc *pushConsumer) resetRetryAndNamespace(subMsgs []*primitive.MessageExt) {
+ groupTopic := internal.RetryGroupTopicPrefix + pc.consumerGroup
+ beginTime := time.Now()
+ for idx := range subMsgs {
+ msg := subMsgs[idx]
+ retryTopic := msg.GetProperty(primitive.PropertyRetryTopic)
+ if retryTopic == "" && groupTopic == msg.Topic {
+ msg.Topic = retryTopic
+ }
+ subMsgs[idx].WithProperty(primitive.PropertyConsumeStartTime, strconv.FormatInt(
+ beginTime.UnixNano()/int64(time.Millisecond), 10))
+ }
+}
+
+func (pc *pushConsumer) consumeMessageCurrently(pq *processQueue, mq *primitive.MessageQueue) {
+ msgs := pq.getMessages()
+ if msgs == nil {
+ return
+ }
+ for count := 0; count < len(msgs); count++ {
+ var subMsgs []*primitive.MessageExt
+ if count+pc.option.ConsumeMessageBatchMaxSize > len(msgs) {
+ subMsgs = msgs[count:]
+ count = len(msgs)
+ } else {
+ next := count + pc.option.ConsumeMessageBatchMaxSize
+ subMsgs = msgs[count:next]
+ count = next - 1
+ }
+ go primitive.WithRecover(func() {
+ RETRY:
+ if pq.IsDroppd() {
+ rlog.Info("the message queue not be able to consume, because it was dropped", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq.String(),
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ })
+ return
+ }
+
+ beginTime := time.Now()
+ pc.resetRetryAndNamespace(subMsgs)
+ var result ConsumeResult
+
+ var err error
+ msgCtx := &primitive.ConsumeMessageContext{
+ Properties: make(map[string]string),
+ ConsumerGroup: pc.consumerGroup,
+ MQ: mq,
+ Msgs: msgs,
+ }
+ ctx := context.Background()
+ ctx = primitive.WithConsumerCtx(ctx, msgCtx)
+ ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
+ concurrentCtx := primitive.NewConsumeConcurrentlyContext()
+ concurrentCtx.MQ = *mq
+ ctx = primitive.WithConcurrentlyCtx(ctx, concurrentCtx)
+
+ result, err = pc.consumeInner(ctx, subMsgs)
+
+ consumeRT := time.Now().Sub(beginTime)
+ if err != nil {
+ msgCtx.Properties[primitive.PropCtxType] = string(primitive.ExceptionReturn)
+ } else if consumeRT >= pc.option.ConsumeTimeout {
+ msgCtx.Properties[primitive.PropCtxType] = string(primitive.TimeoutReturn)
+ } else if result == ConsumeSuccess {
+ msgCtx.Properties[primitive.PropCtxType] = string(primitive.SuccessReturn)
+ } else if result == ConsumeRetryLater {
+ msgCtx.Properties[primitive.PropCtxType] = string(primitive.FailedReturn)
+ }
+
+ increaseConsumeRT(pc.consumerGroup, mq.Topic, int64(consumeRT/time.Millisecond))
+
+ if !pq.IsDroppd() {
+ msgBackFailed := make([]*primitive.MessageExt, 0)
+ if result == ConsumeSuccess {
+ increaseConsumeOKTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
+ } else {
+ increaseConsumeFailedTPS(pc.consumerGroup, mq.Topic, len(subMsgs))
+ if pc.model == BroadCasting {
+ for i := 0; i < len(msgs); i++ {
+ rlog.Warning("BROADCASTING, the message consume failed, drop it", map[string]interface{}{
+ "message": subMsgs[i],
+ })
+ }
+ } else {
+ for i := 0; i < len(msgs); i++ {
+ msg := msgs[i]
+ if !pc.sendMessageBack(mq.BrokerName, msg, concurrentCtx.DelayLevelWhenNextConsume) {
+ msg.ReconsumeTimes += 1
+ msgBackFailed = append(msgBackFailed, msg)
+ }
+ }
+ }
+ }
+
+ offset := pq.removeMessage(subMsgs...)
+
+ if offset >= 0 && !pq.IsDroppd() {
+ pc.storage.update(mq, int64(offset), true)
+ }
+ if len(msgBackFailed) > 0 {
+ subMsgs = msgBackFailed
+ time.Sleep(5 * time.Second)
+ goto RETRY
+ }
+ } else {
+ rlog.Warning("processQueue is dropped without process consume result.", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq,
+ "message": msgs,
+ })
+ }
+ })
+ }
+}
+
+func (pc *pushConsumer) consumeMessageOrderly(pq *processQueue, mq *primitive.MessageQueue) {
+ if pq.IsDroppd() {
+ rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ return
+ }
+
+ lock := pc.queueLock.fetchLock(*mq)
+ lock.Lock()
+ defer lock.Unlock()
+ if pc.model == BroadCasting || (pq.IsLock() && !pq.isLockExpired()) {
+ beginTime := time.Now()
+
+ continueConsume := true
+ for continueConsume {
+ if pq.IsDroppd() {
+ rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ break
+ }
+ if pc.model == Clustering {
+ if !pq.IsLock() {
+ rlog.Warning("the message queue not locked, so consume later", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ pc.tryLockLaterAndReconsume(mq, 10)
+ return
+ }
+ if pq.isLockExpired() {
+ rlog.Warning("the message queue lock expired, so consume later", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ pc.tryLockLaterAndReconsume(mq, 10)
+ return
+ }
+ }
+ interval := time.Now().Sub(beginTime)
+ if interval > pc.option.MaxTimeConsumeContinuously {
+ time.Sleep(10 * time.Millisecond)
+ return
+ }
+ batchSize := pc.option.ConsumeMessageBatchMaxSize
+ msgs := pq.takeMessages(batchSize)
+
+ pc.resetRetryAndNamespace(msgs)
+
+ if len(msgs) == 0 {
+ continueConsume = false
+ break
+ }
+
+ // TODO: add message consumer hook
+ beginTime = time.Now()
+
+ ctx := context.Background()
+ msgCtx := &primitive.ConsumeMessageContext{
+ Properties: make(map[string]string),
+ ConsumerGroup: pc.consumerGroup,
+ MQ: mq,
+ Msgs: msgs,
+ }
+ ctx = primitive.WithConsumerCtx(ctx, msgCtx)
+ ctx = primitive.WithMethod(ctx, primitive.ConsumerPush)
+
+ orderlyCtx := primitive.NewConsumeOrderlyContext()
+ orderlyCtx.MQ = *mq
+ ctx = primitive.WithOrderlyCtx(ctx, orderlyCtx)
+
+ pq.lockConsume.Lock()
+ result, _ := pc.consumeInner(ctx, msgs)
+ pq.lockConsume.Unlock()
+
+ if result == Rollback || result == SuspendCurrentQueueAMoment {
+ rlog.Warning("consumeMessage Orderly return not OK", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: pc.consumerGroup,
+ "messages": msgs,
+ rlog.LogKeyMessageQueue: mq,
+ })
+ }
+
+ // jsut put consumeResult in consumerMessageCtx
+ //interval = time.Now().Sub(beginTime)
+ //consumeReult := SuccessReturn
+ //if interval > pc.option.ConsumeTimeout {
+ // consumeReult = TimeoutReturn
+ //} else if SuspendCurrentQueueAMoment == result {
+ // consumeReult = FailedReturn
+ //} else if ConsumeSuccess == result {
+ // consumeReult = SuccessReturn
+ //}
+
+ // process result
+ commitOffset := int64(-1)
+ if pc.option.AutoCommit {
+ switch result {
+ case Commit, Rollback:
+ rlog.Warning("the message queue consume result is illegal, we think you want to ack these message: %v", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq,
+ })
+ case ConsumeSuccess:
+ commitOffset = pq.commit()
+ case SuspendCurrentQueueAMoment:
+ if pc.checkReconsumeTimes(msgs) {
+ pq.putMessage(msgs...)
+ time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
+ continueConsume = false
+ } else {
+ commitOffset = pq.commit()
+ }
+ default:
+ }
+ } else {
+ switch result {
+ case ConsumeSuccess:
+ case Commit:
+ commitOffset = pq.commit()
+ case Rollback:
+ // pq.rollback
+ time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
+ continueConsume = false
+ case SuspendCurrentQueueAMoment:
+ if pc.checkReconsumeTimes(msgs) {
+ time.Sleep(time.Duration(orderlyCtx.SuspendCurrentQueueTimeMillis) * time.Millisecond)
+ continueConsume = false
+ }
+ default:
+ }
+ }
+ if commitOffset > 0 && !pq.IsDroppd() {
+ _ = pc.updateOffset(mq, commitOffset)
+ }
+ }
+ } else {
+ if pq.IsDroppd() {
+ rlog.Warning("the message queue not be able to consume, because it's dropped.", map[string]interface{}{
+ rlog.LogKeyMessageQueue: mq.String(),
+ })
+ }
+ pc.tryLockLaterAndReconsume(mq, 100)
+ }
+}
+
+func (pc *pushConsumer) checkReconsumeTimes(msgs []*primitive.MessageExt) bool {
+ suspend := false
+ if len(msgs) != 0 {
+ maxReconsumeTimes := pc.getOrderlyMaxReconsumeTimes()
+ for _, msg := range msgs {
+ if msg.ReconsumeTimes > maxReconsumeTimes {
+ rlog.Warning(fmt.Sprintf("msg will be send to retry topic due to ReconsumeTimes > %d, \n", maxReconsumeTimes), nil)
+ msg.WithProperty("RECONSUME_TIME", strconv.Itoa(int(msg.ReconsumeTimes)))
+ if !pc.sendMessageBack("", msg, -1) {
+ suspend = true
+ msg.ReconsumeTimes += 1
+ }
+ } else {
+ suspend = true
+ msg.ReconsumeTimes += 1
+ }
+ }
+ }
+ return suspend
+}
+
+func (pc *pushConsumer) getOrderlyMaxReconsumeTimes() int32 {
+ if pc.option.MaxReconsumeTimes == -1 {
+ return math.MaxInt32
+ } else {
+ return pc.option.MaxReconsumeTimes
+ }
+}
+
+func (pc *pushConsumer) getMaxReconsumeTimes() int32 {
+ if pc.option.MaxReconsumeTimes == -1 {
+ return 16
+ } else {
+ return pc.option.MaxReconsumeTimes
+ }
+}
+
+func (pc *pushConsumer) tryLockLaterAndReconsume(mq *primitive.MessageQueue, delay int64) {
+ time.Sleep(time.Duration(delay) * time.Millisecond)
+ if pc.lock(mq) == true {
+ pc.submitConsumeRequestLater(10)
+ } else {
+ pc.submitConsumeRequestLater(3000)
+ }
+}
+
+func (pc *pushConsumer) submitConsumeRequestLater(suspendTimeMillis int64) {
+ if suspendTimeMillis == -1 {
+ suspendTimeMillis = int64(pc.option.SuspendCurrentQueueTimeMillis / time.Millisecond)
+ }
+ if suspendTimeMillis < 10 {
+ suspendTimeMillis = 10
+ } else if suspendTimeMillis > 30000 {
+ suspendTimeMillis = 30000
+ }
+ time.Sleep(time.Duration(suspendTimeMillis) * time.Millisecond)
+}
diff --git a/consumer/push_consumer_test.go b/consumer/push_consumer_test.go
new file mode 100644
index 0000000..7cb5fca
--- /dev/null
+++ b/consumer/push_consumer_test.go
@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/golang/mock/gomock"
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+func mockB4Start(c *pushConsumer) {
+ c.topicSubscribeInfoTable.Store("TopicTest", []*primitive.MessageQueue{})
+}
+
+func TestStart(t *testing.T) {
+ Convey("test Start method", t, func() {
+ c, _ := NewPushConsumer(
+ WithGroupName("testGroup"),
+ WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ WithConsumerModel(BroadCasting),
+ )
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+
+ client := internal.NewMockRMQClient(ctrl)
+ c.client = client
+
+ err := c.Subscribe("TopicTest", MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return ConsumeSuccess, nil
+ })
+
+ client.EXPECT().ClientID().Return("127.0.0.1@DEFAULT")
+ client.EXPECT().Start().Return()
+ client.EXPECT().RegisterConsumer(gomock.Any(), gomock.Any()).Return(nil)
+ client.EXPECT().UpdateTopicRouteInfo().AnyTimes().Return()
+
+ Convey("test topic route info not found", func() {
+ client.EXPECT().Shutdown().Return()
+ err = c.Start()
+ So(err.Error(), ShouldContainSubstring, "route info not found")
+ })
+
+ Convey("test topic route info found", func() {
+ client.EXPECT().RebalanceImmediately().Return()
+ client.EXPECT().CheckClientInBroker().Return()
+ client.EXPECT().SendHeartbeatToAllBrokerWithLock().Return()
+ mockB4Start(c)
+ err = c.Start()
+ So(err, ShouldBeNil)
+ })
+ })
+}
diff --git a/consumer/statistics.go b/consumer/statistics.go
new file mode 100644
index 0000000..aae5f89
--- /dev/null
+++ b/consumer/statistics.go
@@ -0,0 +1,483 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "container/list"
+ "fmt"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+var (
+ csListLock sync.Mutex
+ closeOnce sync.Once
+
+ topicAndGroupConsumeOKTPS *statsItemSet
+ topicAndGroupConsumeRT *statsItemSet
+ topicAndGroupConsumeFailedTPS *statsItemSet
+ topicAndGroupPullTPS *statsItemSet
+ topicAndGroupPullRT *statsItemSet
+)
+
+func init() {
+ topicAndGroupConsumeOKTPS = newStatsItemSet("CONSUME_OK_TPS")
+ topicAndGroupConsumeRT = newStatsItemSet("CONSUME_RT")
+ topicAndGroupConsumeFailedTPS = newStatsItemSet("CONSUME_FAILED_TPS")
+ topicAndGroupPullTPS = newStatsItemSet("PULL_TPS")
+ topicAndGroupPullRT = newStatsItemSet("PULL_RT")
+}
+
+type ConsumeStatus struct {
+ PullRT float64
+ PullTPS float64
+ ConsumeRT float64
+ ConsumeOKTPS float64
+ ConsumeFailedTPS float64
+ ConsumeFailedMsgs int64
+}
+
+func increasePullRT(group, topic string, rt int64) {
+ topicAndGroupPullRT.addValue(topic+"@"+group, rt, 1)
+}
+
+func increasePullTPS(group, topic string, msgs int) {
+ topicAndGroupPullTPS.addValue(topic+"@"+group, int64(msgs), 1)
+}
+
+func increaseConsumeRT(group, topic string, rt int64) {
+ topicAndGroupConsumeRT.addValue(topic+"@"+group, rt, 1)
+}
+
+func increaseConsumeOKTPS(group, topic string, msgs int) {
+ topicAndGroupConsumeOKTPS.addValue(topic+"@"+group, int64(msgs), 1)
+}
+
+func increaseConsumeFailedTPS(group, topic string, msgs int) {
+ topicAndGroupConsumeFailedTPS.addValue(topic+"@"+group, int64(msgs), 1)
+}
+
+func GetConsumeStatus(group, topic string) ConsumeStatus {
+ cs := ConsumeStatus{}
+ ss := getPullRT(group, topic)
+ cs.PullTPS = ss.tps
+
+ ss = getPullTPS(group, topic)
+ cs.PullTPS = ss.tps
+
+ ss = getConsumeRT(group, topic)
+ cs.ConsumeRT = ss.avgpt
+
+ ss = getConsumeOKTPS(group, topic)
+ cs.ConsumeOKTPS = ss.tps
+
+ ss = getConsumeFailedTPS(group, topic)
+
+ cs.ConsumeFailedTPS = ss.tps
+
+ ss = topicAndGroupConsumeFailedTPS.getStatsDataInHour(topic + "@" + group)
+ cs.ConsumeFailedMsgs = ss.sum
+ return cs
+}
+
+func ShutDownStatis() {
+ closeOnce.Do(func() {
+ close(topicAndGroupConsumeOKTPS.closed)
+ close(topicAndGroupConsumeRT.closed)
+ close(topicAndGroupConsumeFailedTPS.closed)
+ close(topicAndGroupPullTPS.closed)
+ close(topicAndGroupPullRT.closed)
+ })
+}
+
+func getPullRT(group, topic string) statsSnapshot {
+ return topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
+}
+
+func getPullTPS(group, topic string) statsSnapshot {
+ return topicAndGroupPullTPS.getStatsDataInMinute(topic + "@" + group)
+}
+
+func getConsumeRT(group, topic string) statsSnapshot {
+ ss := topicAndGroupPullRT.getStatsDataInMinute(topic + "@" + group)
+ if ss.sum == 0 {
+ return topicAndGroupConsumeRT.getStatsDataInHour(topic + "@" + group)
+ }
+ return ss
+}
+
+func getConsumeOKTPS(group, topic string) statsSnapshot {
+ return topicAndGroupConsumeOKTPS.getStatsDataInMinute(topic + "@" + group)
+}
+
+func getConsumeFailedTPS(group, topic string) statsSnapshot {
+ return topicAndGroupConsumeFailedTPS.getStatsDataInMinute(topic + "@" + group)
+}
+
+type statsItemSet struct {
+ statsName string
+ statsItemTable sync.Map
+ closed chan struct{}
+}
+
+func newStatsItemSet(statsName string) *statsItemSet {
+ sis := &statsItemSet{
+ statsName: statsName,
+ closed: make(chan struct{}),
+ }
+ sis.init()
+ return sis
+}
+
+func (sis *statsItemSet) init() {
+ go primitive.WithRecover(func() {
+ ticker := time.NewTicker(10 * time.Second)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-sis.closed:
+ return
+ case <-ticker.C:
+ sis.samplingInSeconds()
+
+ }
+ }
+ })
+
+ go primitive.WithRecover(func() {
+ ticker := time.NewTicker(10 * time.Minute)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-sis.closed:
+ return
+ case <-ticker.C:
+ sis.samplingInMinutes()
+ }
+ }
+ })
+
+ go primitive.WithRecover(func() {
+ ticker := time.NewTicker(time.Hour)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-sis.closed:
+ return
+ case <-ticker.C:
+ sis.samplingInHour()
+ }
+ }
+ })
+
+ go primitive.WithRecover(func() {
+ time.Sleep(nextMinutesTime().Sub(time.Now()))
+ ticker := time.NewTicker(time.Minute)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-sis.closed:
+ return
+ case <-ticker.C:
+ sis.printAtMinutes()
+ }
+ }
+ })
+
+ go primitive.WithRecover(func() {
+ time.Sleep(nextHourTime().Sub(time.Now()))
+ ticker := time.NewTicker(time.Hour)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-sis.closed:
+ return
+ case <-ticker.C:
+ sis.printAtHour()
+ }
+ }
+ })
+
+ go primitive.WithRecover(func() {
+ time.Sleep(nextMonthTime().Sub(time.Now()))
+ ticker := time.NewTicker(24 * time.Hour)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-sis.closed:
+ return
+ case <-ticker.C:
+ sis.printAtDay()
+ }
+ }
+ })
+}
+
+func (sis *statsItemSet) samplingInSeconds() {
+ sis.statsItemTable.Range(func(key, value interface{}) bool {
+ si := value.(*statsItem)
+ si.samplingInSeconds()
+ return true
+ })
+}
+
+func (sis *statsItemSet) samplingInMinutes() {
+ sis.statsItemTable.Range(func(key, value interface{}) bool {
+ si := value.(*statsItem)
+ si.samplingInMinutes()
+ return true
+ })
+}
+
+func (sis *statsItemSet) samplingInHour() {
+ sis.statsItemTable.Range(func(key, value interface{}) bool {
+ si := value.(*statsItem)
+ si.samplingInHour()
+ return true
+ })
+}
+
+func (sis *statsItemSet) printAtMinutes() {
+ sis.statsItemTable.Range(func(key, value interface{}) bool {
+ si := value.(*statsItem)
+ si.printAtMinutes()
+ return true
+ })
+}
+
+func (sis *statsItemSet) printAtHour() {
+ sis.statsItemTable.Range(func(key, value interface{}) bool {
+ si := value.(*statsItem)
+ si.printAtHour()
+ return true
+ })
+}
+
+func (sis *statsItemSet) printAtDay() {
+ sis.statsItemTable.Range(func(key, value interface{}) bool {
+ si := value.(*statsItem)
+ si.printAtDay()
+ return true
+ })
+}
+
+func (sis *statsItemSet) addValue(key string, incValue, incTimes int64) {
+ si := sis.getAndCreateStateItem(key)
+ atomic.AddInt64(&si.value, incValue)
+ atomic.AddInt64(&si.times, incTimes)
+}
+
+func (sis *statsItemSet) getAndCreateStateItem(key string) *statsItem {
+ if val, ok := sis.statsItemTable.Load(key); ok {
+ return val.(*statsItem)
+ } else {
+ si := newStatsItem(sis.statsName, key)
+ sis.statsItemTable.Store(key, si)
+ return si
+ }
+}
+
+func (sis *statsItemSet) getStatsDataInMinute(key string) statsSnapshot {
+ if val, ok := sis.statsItemTable.Load(key); ok {
+ si := val.(*statsItem)
+ return si.getStatsDataInMinute()
+ }
+ return statsSnapshot{}
+}
+
+func (sis *statsItemSet) getStatsDataInHour(key string) statsSnapshot {
+ if val, ok := sis.statsItemTable.Load(key); ok {
+ si := val.(*statsItem)
+ return si.getStatsDataInHour()
+ }
+ return statsSnapshot{}
+}
+
+func (sis *statsItemSet) getStatsDataInDay(key string) statsSnapshot {
+ if val, ok := sis.statsItemTable.Load(key); ok {
+ si := val.(*statsItem)
+ return si.getStatsDataInDay()
+ }
+ return statsSnapshot{}
+}
+
+func (sis *statsItemSet) getStatsItem(key string) *statsItem {
+ val, _ := sis.statsItemTable.Load(key)
+ return val.(*statsItem)
+}
+
+type statsItem struct {
+ value int64
+ times int64
+ csListMinute *list.List
+ csListHour *list.List
+ csListDay *list.List
+ statsName string
+ statsKey string
+ csListMinuteLock sync.Mutex
+ csListHourLock sync.Mutex
+ csListDayLock sync.Mutex
+}
+
+func (si *statsItem) getStatsDataInMinute() statsSnapshot {
+ return computeStatsData(si.csListMinute)
+}
+
+func (si *statsItem) getStatsDataInHour() statsSnapshot {
+ return computeStatsData(si.csListHour)
+}
+
+func (si *statsItem) getStatsDataInDay() statsSnapshot {
+ return computeStatsData(si.csListDay)
+}
+
+func newStatsItem(statsName, statsKey string) *statsItem {
+ return &statsItem{
+ statsName: statsName,
+ statsKey: statsKey,
+ csListMinute: list.New(),
+ csListHour: list.New(),
+ csListDay: list.New(),
+ }
+}
+
+func (si *statsItem) samplingInSeconds() {
+ si.csListMinuteLock.Lock()
+ defer si.csListMinuteLock.Unlock()
+ si.csListMinute.PushBack(callSnapshot{
+ timestamp: time.Now().Unix() * 1000,
+ time: atomic.LoadInt64(&si.times),
+ value: atomic.LoadInt64(&si.value),
+ })
+ if si.csListMinute.Len() > 7 {
+ si.csListMinute.Remove(si.csListMinute.Front())
+ }
+}
+
+func (si *statsItem) samplingInMinutes() {
+ si.csListHourLock.Lock()
+ defer si.csListHourLock.Unlock()
+ si.csListHour.PushBack(callSnapshot{
+ timestamp: time.Now().Unix() * 1000,
+ time: atomic.LoadInt64(&si.times),
+ value: atomic.LoadInt64(&si.value),
+ })
+ if si.csListHour.Len() > 7 {
+ si.csListHour.Remove(si.csListHour.Front())
+ }
+}
+
+func (si *statsItem) samplingInHour() {
+ si.csListDayLock.Lock()
+ defer si.csListDayLock.Unlock()
+ si.csListDay.PushBack(callSnapshot{
+ timestamp: time.Now().Unix() * 1000,
+ time: atomic.LoadInt64(&si.times),
+ value: atomic.LoadInt64(&si.value),
+ })
+ if si.csListDay.Len() > 25 {
+ si.csListHour.Remove(si.csListDay.Front())
+ }
+}
+
+func (si *statsItem) printAtMinutes() {
+ ss := computeStatsData(si.csListMinute)
+ rlog.Info("Stats In One Minute, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
+ "statsName": si.statsName,
+ "statsKey": si.statsKey,
+ "SUM": ss.sum,
+ "TPS": fmt.Sprintf("%.2f", ss.tps),
+ "AVGPT": ss.avgpt,
+ })
+}
+
+func (si *statsItem) printAtHour() {
+ ss := computeStatsData(si.csListHour)
+ rlog.Info("Stats In One Hour, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
+ "statsName": si.statsName,
+ "statsKey": si.statsKey,
+ "SUM": ss.sum,
+ "TPS": fmt.Sprintf("%.2f", ss.tps),
+ "AVGPT": ss.avgpt,
+ })
+}
+
+func (si *statsItem) printAtDay() {
+ ss := computeStatsData(si.csListDay)
+ rlog.Info("Stats In One Day, SUM: %d TPS: AVGPT: %.2f", map[string]interface{}{
+ "statsName": si.statsName,
+ "statsKey": si.statsKey,
+ "SUM": ss.sum,
+ "TPS": fmt.Sprintf("%.2f", ss.tps),
+ "AVGPT": ss.avgpt,
+ })
+}
+
+func nextMinutesTime() time.Time {
+ now := time.Now()
+ m, _ := time.ParseDuration("1m")
+ return now.Add(m)
+}
+
+func nextHourTime() time.Time {
+ now := time.Now()
+ m, _ := time.ParseDuration("1h")
+ return now.Add(m)
+}
+
+func nextMonthTime() time.Time {
+ now := time.Now()
+ return now.AddDate(0, 1, 0)
+}
+
+func computeStatsData(csList *list.List) statsSnapshot {
+ csListLock.Lock()
+ defer csListLock.Unlock()
+ tps, avgpt, sum := 0.0, 0.0, int64(0)
+ if csList.Len() > 0 {
+ first := csList.Front().Value.(callSnapshot)
+ last := csList.Back().Value.(callSnapshot)
+ sum = last.value - first.value
+ tps = float64(sum*1000.0) / float64(last.timestamp-first.timestamp)
+ timesDiff := last.time - first.time
+ if timesDiff > 0 {
+ avgpt = float64(sum*1.0) / float64(timesDiff)
+ }
+ }
+ return statsSnapshot{
+ tps: tps,
+ avgpt: avgpt,
+ sum: sum,
+ }
+}
+
+type callSnapshot struct {
+ timestamp int64
+ time int64
+ value int64
+}
+
+type statsSnapshot struct {
+ sum int64
+ tps float64
+ avgpt float64
+}
diff --git a/consumer/statistics_test.go b/consumer/statistics_test.go
new file mode 100644
index 0000000..f70ed0b
--- /dev/null
+++ b/consumer/statistics_test.go
@@ -0,0 +1,209 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "testing"
+ "time"
+)
+
+func almostEqual(a, b float64) bool {
+ diff := abs(a - b)
+ return diff/a < 0.01
+}
+
+func abs(a float64) float64 {
+ if a > 0 {
+ return a
+ }
+ return -a
+}
+
+func TestNextMinuteTime(t *testing.T) {
+ nextMinute := nextMinutesTime()
+ minuteElapse := nextMinute.Sub(time.Now()).Minutes()
+ if !almostEqual(minuteElapse, 1.0) {
+ t.Errorf("wrong next one minute. want=%f, got=%f", 1.0, minuteElapse)
+ }
+}
+
+func TestNextHourTime(t *testing.T) {
+ nextHour := nextHourTime()
+ hourElapse := nextHour.Sub(time.Now()).Hours()
+ if !almostEqual(hourElapse, 1.0) {
+ t.Errorf("wrong next one hour. want=%f, got=%f", 1.0, hourElapse)
+ }
+}
+
+func TestIncreasePullRTGetPullRT(t *testing.T) {
+ ShutDownStatis()
+
+ tests := []struct {
+ RT int64
+ ExpectSum int64
+ }{
+ {1, 0},
+ {1, 1},
+ {1, 2},
+ {1, 3},
+ {1, 4},
+ {1, 5},
+ {1, 6},
+ {1, 6},
+ }
+ for _, tt := range tests {
+ increasePullRT("rocketmq", "default", tt.RT)
+ topicAndGroupPullRT.samplingInSeconds()
+ snapshot := getPullRT("rocketmq", "default")
+ if snapshot.sum != tt.ExpectSum {
+ t.Errorf("wrong Pull RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
+ }
+ }
+}
+
+//func TestIncreaseConsumeRTGetConsumeRT(t *testing.T) {
+// ShutDownStatis()
+// tests := []struct {
+// RT int64
+// ExpectSum int64
+// }{
+// {1, 0},
+// {1, 1},
+// {1, 2},
+// {1, 3},
+// {1, 4},
+// {1, 5},
+// {1, 6},
+// {1, 6},
+// }
+// for _, tt := range tests {
+// increaseConsumeRT("rocketmq", "default", tt.RT)
+// topicAndGroupConsumeRT.samplingInMinutes()
+// snapshot := getConsumeRT("rocketmq", "default")
+// if snapshot.sum != tt.ExpectSum {
+// t.Errorf("wrong consume RT sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
+// }
+// }
+//}
+
+func TestIncreasePullTPSGetPullTPS(t *testing.T) {
+ ShutDownStatis()
+ tests := []struct {
+ RT int
+ ExpectSum int64
+ }{
+ {1, 0},
+ {1, 1},
+ {1, 2},
+ {1, 3},
+ {1, 4},
+ {1, 5},
+ {1, 6},
+ {1, 6},
+ }
+ for _, tt := range tests {
+ increasePullTPS("rocketmq", "default", tt.RT)
+ topicAndGroupPullTPS.samplingInSeconds()
+ snapshot := getPullTPS("rocketmq", "default")
+ if snapshot.sum != tt.ExpectSum {
+ t.Errorf("wrong Pull TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
+ }
+ }
+}
+
+func TestIncreaseConsumeOKTPSGetConsumeOKTPS(t *testing.T) {
+ ShutDownStatis()
+ tests := []struct {
+ RT int
+ ExpectSum int64
+ }{
+ {1, 0},
+ {1, 1},
+ {1, 2},
+ {1, 3},
+ {1, 4},
+ {1, 5},
+ {1, 6},
+ {1, 6},
+ }
+ for _, tt := range tests {
+ increaseConsumeOKTPS("rocketmq", "default", tt.RT)
+ topicAndGroupConsumeOKTPS.samplingInSeconds()
+ snapshot := getConsumeOKTPS("rocketmq", "default")
+ if snapshot.sum != tt.ExpectSum {
+ t.Errorf("wrong Consume OK TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
+ }
+ }
+}
+
+func TestIncreaseConsumeFailedTPSGetConsumeFailedTPS(t *testing.T) {
+ ShutDownStatis()
+ tests := []struct {
+ RT int
+ ExpectSum int64
+ }{
+ {1, 0},
+ {1, 1},
+ {1, 2},
+ {1, 3},
+ {1, 4},
+ {1, 5},
+ {1, 6},
+ {1, 6},
+ }
+ for _, tt := range tests {
+ increaseConsumeFailedTPS("rocketmq", "default", tt.RT)
+ topicAndGroupConsumeFailedTPS.samplingInSeconds()
+ snapshot := getConsumeFailedTPS("rocketmq", "default")
+ if snapshot.sum != tt.ExpectSum {
+ t.Errorf("wrong Consume Failed TPS sum. want=%d, got=%d", tt.ExpectSum, snapshot.sum)
+ }
+ }
+}
+
+func TestGetConsumeStatus(t *testing.T) {
+ ShutDownStatis()
+ group, topic := "rocketmq", "default"
+
+ tests := []struct {
+ RT int
+ ExpectFailMessage int64
+ }{
+ {1, 0},
+ {1, 1},
+ {1, 2},
+ {1, 3},
+ {1, 4},
+ }
+ for _, tt := range tests {
+ increasePullRT(group, topic, int64(tt.RT))
+ increasePullTPS(group, topic, tt.RT)
+ increaseConsumeRT(group, topic, int64(tt.RT))
+ increaseConsumeOKTPS(group, topic, tt.RT)
+ increaseConsumeFailedTPS(group, topic, tt.RT)
+ topicAndGroupPullRT.samplingInSeconds()
+ topicAndGroupPullTPS.samplingInSeconds()
+ topicAndGroupConsumeRT.samplingInMinutes()
+ topicAndGroupConsumeOKTPS.samplingInSeconds()
+ topicAndGroupConsumeFailedTPS.samplingInMinutes()
+ status := GetConsumeStatus(group, topic)
+ if status.ConsumeFailedMsgs != tt.ExpectFailMessage {
+ t.Errorf("wrong ConsumeFailedMsg. want=0, got=%d", status.ConsumeFailedMsgs)
+ }
+ }
+}
diff --git a/consumer/strategy.go b/consumer/strategy.go
new file mode 100644
index 0000000..2af48de
--- /dev/null
+++ b/consumer/strategy.go
@@ -0,0 +1,247 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "strings"
+
+ "stathat.com/c/consistent"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+// Strategy Algorithm for message allocating between consumers
+// An allocate strategy proxy for based on machine room nearside priority. An actual allocate strategy can be
+// specified.
+//
+// If any consumer is alive in a machine room, the message queue of the broker which is deployed in the same machine
+// should only be allocated to those. Otherwise, those message queues can be shared along all consumers since there are
+// no alive consumer to monopolize them.
+//
+// Average Hashing queue algorithm
+// Cycle average Hashing queue algorithm
+// Use Message QueueID specified
+// Computer room Hashing queue algorithm, such as Alipay logic room
+// Consistent Hashing queue algorithm
+
+type AllocateStrategy func(string, string, []*primitive.MessageQueue, []string) []*primitive.MessageQueue
+
+func AllocateByAveragely(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
+ cidAll []string) []*primitive.MessageQueue {
+ if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
+ return nil
+ }
+
+ var (
+ find bool
+ index int
+ )
+ for idx := range cidAll {
+ if cidAll[idx] == currentCID {
+ find = true
+ index = idx
+ break
+ }
+ }
+ if !find {
+ rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: consumerGroup,
+ "consumerId": currentCID,
+ "cidAll": cidAll,
+ })
+ return nil
+ }
+
+ mqSize := len(mqAll)
+ cidSize := len(cidAll)
+ mod := mqSize % cidSize
+
+ var averageSize int
+ if mqSize <= cidSize {
+ averageSize = 1
+ } else {
+ if mod > 0 && index < mod {
+ averageSize = mqSize/cidSize + 1
+ } else {
+ averageSize = mqSize / cidSize
+ }
+ }
+
+ var startIndex int
+ if mod > 0 && index < mod {
+ startIndex = index * averageSize
+ } else {
+ startIndex = index*averageSize + mod
+ }
+
+ num := utils.MinInt(averageSize, mqSize-startIndex)
+ result := make([]*primitive.MessageQueue, 0)
+ for i := 0; i < num; i++ {
+ result = append(result, mqAll[(startIndex+i)%mqSize])
+ }
+ return result
+}
+
+func AllocateByAveragelyCircle(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
+ cidAll []string) []*primitive.MessageQueue {
+ if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
+ return nil
+ }
+
+ var (
+ find bool
+ index int
+ )
+ for idx := range cidAll {
+ if cidAll[idx] == currentCID {
+ find = true
+ index = idx
+ break
+ }
+ }
+ if !find {
+ rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: consumerGroup,
+ "consumerId": currentCID,
+ "cidAll": cidAll,
+ })
+ return nil
+ }
+
+ result := make([]*primitive.MessageQueue, 0)
+ for i := index; i < len(mqAll); i++ {
+ if i%len(cidAll) == index {
+ result = append(result, mqAll[i])
+ }
+ }
+ return result
+}
+
+// TODO
+func AllocateByMachineNearby(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue,
+ cidAll []string) []*primitive.MessageQueue {
+ return AllocateByAveragely(consumerGroup, currentCID, mqAll, cidAll)
+}
+
+func AllocateByConfig(list []*primitive.MessageQueue) AllocateStrategy {
+ return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
+ return list
+ }
+}
+
+func AllocateByMachineRoom(consumeridcs []string) AllocateStrategy {
+ return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
+ if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
+ return nil
+ }
+
+ var (
+ find bool
+ index int
+ )
+ for idx := range cidAll {
+ if cidAll[idx] == currentCID {
+ find = true
+ index = idx
+ break
+ }
+ }
+ if !find {
+ rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: consumerGroup,
+ "consumerId": currentCID,
+ "cidAll": cidAll,
+ })
+ return nil
+ }
+
+ var premqAll []*primitive.MessageQueue
+ for _, mq := range mqAll {
+ temp := strings.Split(mq.BrokerName, "@")
+ if len(temp) == 2 {
+ for _, idc := range consumeridcs {
+ if idc == temp[0] {
+ premqAll = append(premqAll, mq)
+ }
+ }
+ }
+ }
+
+ mod := len(premqAll) / len(cidAll)
+ rem := len(premqAll) % len(cidAll)
+ startIndex := mod * index
+ endIndex := startIndex + mod
+
+ result := make([]*primitive.MessageQueue, 0)
+ for i := startIndex; i < endIndex; i++ {
+ result = append(result, mqAll[i])
+ }
+ if rem > index {
+ result = append(result, premqAll[index+mod*len(cidAll)])
+ }
+ return result
+ }
+}
+
+func AllocateByConsistentHash(virtualNodeCnt int) AllocateStrategy {
+ return func(consumerGroup, currentCID string, mqAll []*primitive.MessageQueue, cidAll []string) []*primitive.MessageQueue {
+ if currentCID == "" || len(mqAll) == 0 || len(cidAll) == 0 {
+ return nil
+ }
+
+ var (
+ find bool
+ )
+ for idx := range cidAll {
+ if cidAll[idx] == currentCID {
+ find = true
+ break
+ }
+ }
+ if !find {
+ rlog.Warning("[BUG] ConsumerId not in cidAll", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: consumerGroup,
+ "consumerId": currentCID,
+ "cidAll": cidAll,
+ })
+ return nil
+ }
+
+ c := consistent.New()
+ c.NumberOfReplicas = virtualNodeCnt
+ for _, cid := range cidAll {
+ c.Add(cid)
+ }
+
+ result := make([]*primitive.MessageQueue, 0)
+ for _, mq := range mqAll {
+ clientNode, err := c.Get(mq.String())
+ if err != nil {
+ rlog.Warning("[BUG] AllocateByConsistentHash err: %s", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ if currentCID == clientNode {
+ result = append(result, mq)
+ }
+ }
+ return result
+ }
+}
diff --git a/consumer/strategy_test.go b/consumer/strategy_test.go
new file mode 100644
index 0000000..e66b15c
--- /dev/null
+++ b/consumer/strategy_test.go
@@ -0,0 +1,483 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package consumer
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+func TestAllocateByAveragely(t *testing.T) {
+ Convey("Given message queues with a starting value", t, func() {
+ queues := []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ },
+ {
+ QueueId: 1,
+ },
+ {
+ QueueId: 2,
+ },
+ {
+ QueueId: 3,
+ },
+ {
+ QueueId: 4,
+ },
+ {
+ QueueId: 5,
+ },
+ }
+
+ Convey("When params is empty", func() {
+ result := AllocateByAveragely("testGroup", "", queues, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = AllocateByAveragely("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = AllocateByAveragely("testGroup", "192.168.24.1@default", queues, nil)
+ So(result, ShouldBeNil)
+ })
+
+ type testCase struct {
+ currentCid string
+ mqAll []*primitive.MessageQueue
+ cidAll []string
+ expectedQueue []*primitive.MessageQueue
+ }
+ cases := []testCase{
+ {
+ currentCid: "192.168.24.1@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ },
+ {
+ QueueId: 1,
+ },
+ {
+ QueueId: 2,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 2,
+ },
+ {
+ QueueId: 3,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 2,
+ },
+ {
+ QueueId: 3,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.4@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 5,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.7@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
+ expectedQueue: []*primitive.MessageQueue{},
+ },
+ }
+
+ Convey("the result of AllocateByAveragely should be deep equal expectedQueue", func() {
+ for _, value := range cases {
+ result := AllocateByAveragely("testGroup", value.currentCid, value.mqAll, value.cidAll)
+ So(result, ShouldResemble, value.expectedQueue)
+ }
+ })
+ })
+}
+
+func TestAllocateByAveragelyCircle(t *testing.T) {
+ Convey("Given message queues with a starting value", t, func() {
+ queues := []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ },
+ {
+ QueueId: 1,
+ },
+ {
+ QueueId: 2,
+ },
+ {
+ QueueId: 3,
+ },
+ {
+ QueueId: 4,
+ },
+ {
+ QueueId: 5,
+ },
+ }
+
+ Convey("When params is empty", func() {
+ result := AllocateByAveragelyCircle("testGroup", "", queues, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = AllocateByAveragelyCircle("testGroup", "192.168.24.1@default", queues, nil)
+ So(result, ShouldBeNil)
+ })
+
+ type testCase struct {
+ currentCid string
+ mqAll []*primitive.MessageQueue
+ cidAll []string
+ expectedQueue []*primitive.MessageQueue
+ }
+ cases := []testCase{
+ {
+ currentCid: "192.168.24.1@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ },
+ {
+ QueueId: 2,
+ },
+ {
+ QueueId: 4,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 1,
+ },
+ {
+ QueueId: 4,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 1,
+ },
+ {
+ QueueId: 5,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.4@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 3,
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.7@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
+ expectedQueue: []*primitive.MessageQueue{},
+ },
+ }
+
+ Convey("the result of AllocateByAveragelyCircle should be deep equal expectedQueue", func() {
+ for _, value := range cases {
+ result := AllocateByAveragelyCircle("testGroup", value.currentCid, value.mqAll, value.cidAll)
+ So(result, ShouldResemble, value.expectedQueue)
+ }
+ })
+ })
+}
+
+func TestAllocateByConfig(t *testing.T) {
+ Convey("Given message queues with a starting value", t, func() {
+ queues := []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ },
+ {
+ QueueId: 1,
+ },
+ {
+ QueueId: 2,
+ },
+ {
+ QueueId: 3,
+ },
+ {
+ QueueId: 4,
+ },
+ {
+ QueueId: 5,
+ },
+ }
+
+ strategy := AllocateByConfig(queues)
+ result := strategy("testGroup", "192.168.24.1@default", queues, []string{"192.168.24.1@default", "192.168.24.2@default"})
+ So(result, ShouldResemble, queues)
+ })
+}
+
+func TestAllocateByMachineRoom(t *testing.T) {
+ Convey("Given some consumer IDCs with a starting value", t, func() {
+ idcs := []string{"192.168.24.1", "192.168.24.2"}
+ strategy := AllocateByMachineRoom(idcs)
+
+ queues := []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 1,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 2,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 3,
+ BrokerName: "192.168.24.2@defaultName",
+ },
+ {
+ QueueId: 4,
+ BrokerName: "192.168.24.2@defaultName",
+ },
+ {
+ QueueId: 5,
+ BrokerName: "192.168.24.3@defaultName",
+ },
+ }
+
+ Convey("When params is empty", func() {
+ result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = strategy("testGroup", "192.168.24.1@default", queues, nil)
+ So(result, ShouldBeNil)
+ })
+
+ type testCase struct {
+ currentCid string
+ mqAll []*primitive.MessageQueue
+ cidAll []string
+ expectedQueue []*primitive.MessageQueue
+ }
+ cases := []testCase{
+ {
+ currentCid: "192.168.24.1@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 1,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 4,
+ BrokerName: "192.168.24.2@defaultName",
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 1,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 4,
+ BrokerName: "192.168.24.2@defaultName",
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 1,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.4@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default"},
+ expectedQueue: []*primitive.MessageQueue{
+ {
+ QueueId: 3,
+ BrokerName: "192.168.24.2@defaultName",
+ },
+ },
+ },
+ {
+ currentCid: "192.168.24.7@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default", "192.168.24.4@default", "192.168.24.5@default", "192.168.24.6@default", "192.168.24.7@default"},
+ expectedQueue: []*primitive.MessageQueue{},
+ },
+ }
+
+ Convey("the result of AllocateByMachineRoom should be deep equal expectedQueue", func() {
+ for _, value := range cases {
+ result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
+ So(result, ShouldResemble, value.expectedQueue)
+ }
+ })
+ })
+}
+
+func TestAllocateByConsistentHash(t *testing.T) {
+ Convey("Given virtualNodeCnt with a starting value", t, func() {
+ virtualNodeCnt := 10
+ strategy := AllocateByConsistentHash(virtualNodeCnt)
+
+ queues := []*primitive.MessageQueue{
+ {
+ QueueId: 0,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 1,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 2,
+ BrokerName: "192.168.24.1@defaultName",
+ },
+ {
+ QueueId: 3,
+ BrokerName: "192.168.24.2@defaultName",
+ },
+ {
+ QueueId: 4,
+ BrokerName: "192.168.24.2@defaultName",
+ },
+ {
+ QueueId: 5,
+ BrokerName: "192.168.24.3@defaultName",
+ },
+ }
+
+ Convey("When params is empty", func() {
+ result := strategy("testGroup", "", queues, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = strategy("testGroup", "192.168.24.1@default", nil, []string{"192.168.24.1@default"})
+ So(result, ShouldBeNil)
+
+ result = strategy("testGroup", "192.168.24.1@default", queues, nil)
+ So(result, ShouldBeNil)
+ })
+
+ type testCase struct {
+ currentCid string
+ mqAll []*primitive.MessageQueue
+ cidAll []string
+ }
+ cases := []testCase{
+ {
+ currentCid: "192.168.24.1@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
+ },
+ {
+ currentCid: "192.168.24.3@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default", "192.168.24.3@default"},
+ },
+ {
+ currentCid: "192.168.24.1@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
+ },
+ {
+ currentCid: "192.168.24.2@default",
+ mqAll: queues,
+ cidAll: []string{"192.168.24.1@default", "192.168.24.2@default"},
+ },
+ }
+
+ Convey("observe the result of AllocateByMachineRoom", func() {
+ for _, value := range cases {
+ result := strategy("testGroup", value.currentCid, value.mqAll, value.cidAll)
+ fmt.Printf("\n\n currentCid:%s, cidAll:%s, \n allocateResult:%+v \n", value.currentCid, value.cidAll, result)
+ }
+ })
+ })
+}
diff --git a/core/api.go b/core/api.go
deleted file mode 100644
index 79918f1..0000000
--- a/core/api.go
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-import "fmt"
-
-//Version get go sdk version
-func Version() (version string) {
- return GetVersion()
-}
-
-//ClientConfig save client config
-type ClientConfig struct {
- GroupID string
- NameServer string
- NameServerDomain string
- InstanceName string
- Credentials *SessionCredentials
- LogC *LogConfig
-}
-
-func (config *ClientConfig) String() string {
- // For security, don't print Credentials.
- str := ""
- str = strJoin(str, "GroupId", config.GroupID)
- str = strJoin(str, "NameServer", config.NameServer)
- str = strJoin(str, "NameServerDomain", config.NameServerDomain)
- str = strJoin(str, "InstanceName", config.InstanceName)
-
- if config.LogC != nil {
- str = strJoin(str, "LogConfig", config.LogC.String())
- }
-
- return str
-}
-
-//ProducerModel Common or orderly
-type ProducerModel int
-
-//Different models
-const (
- CommonProducer = ProducerModel(1)
- OrderlyProducer = ProducerModel(2)
- TransProducer = ProducerModel(3)
-)
-
-func (mode ProducerModel) String() string {
- switch mode {
- case CommonProducer:
- return "CommonProducer"
- case OrderlyProducer:
- return "OrderlyProducer"
- case TransProducer:
- return "TransProducer"
- default:
- return "Unknown"
- }
-}
-
-// NewProducer create a new producer with config
-func NewProducer(config *ProducerConfig) (Producer, error) {
- return newDefaultProducer(config)
-}
-
-// ProducerConfig define a producer
-type ProducerConfig struct {
- ClientConfig
- SendMsgTimeout int
- CompressLevel int
- MaxMessageSize int
- ProducerModel ProducerModel
-}
-
-func (config *ProducerConfig) String() string {
- str := "ProducerConfig=[" + config.ClientConfig.String()
-
- if config.SendMsgTimeout > 0 {
- str = strJoin(str, "SendMsgTimeout", config.SendMsgTimeout)
- }
-
- if config.CompressLevel > 0 {
- str = strJoin(str, "CompressLevel", config.CompressLevel)
- }
-
- if config.MaxMessageSize > 0 {
- str = strJoin(str, "MaxMessageSize", config.MaxMessageSize)
- }
- str = strJoin(str, "ProducerModel", config.ProducerModel.String())
- return str + "]"
-}
-
-//Producer define interface
-type Producer interface {
- baseAPI
- // SendMessageSync send a message with sync
- SendMessageSync(msg *Message) (*SendResult, error)
-
- // SendMessageOrderly send the message orderly
- SendMessageOrderly(
- msg *Message,
- selector MessageQueueSelector,
- arg interface{},
- autoRetryTimes int) (*SendResult, error)
-
- // SendMessageOneway send a message with oneway
- SendMessageOneway(msg *Message) error
-
- SendMessageOrderlyByShardingKey(msg *Message, shardingkey string) (*SendResult, error)
-}
-
-// NewTransactionProducer create a new trasaction producer with config
-func NewTransactionProducer(config *ProducerConfig, listener TransactionLocalListener, arg interface{}) (TransactionProducer, error) {
- return newDefaultTransactionProducer(config, listener, arg)
-}
-
-// TransactionLocalListener local listener for transaction message
-type TransactionLocalListener interface {
- Execute(m *Message, arg interface{}) TransactionStatus
- Check(m *MessageExt, arg interface{}) TransactionStatus
-}
-
-// TransactionProducer api for send transaction message
-type TransactionProducer interface {
- baseAPI
- // send a transaction message with sync
- SendMessageTransaction(msg *Message, arg interface{}) (*SendResult, error)
-}
-
-// NewPushConsumer create a new consumer with config.
-func NewPushConsumer(config *PushConsumerConfig) (PushConsumer, error) {
- return newPushConsumer(config)
-}
-
-//MessageModel Clustering or BroadCasting
-type MessageModel int
-
-//MessageModel
-const (
- BroadCasting = MessageModel(1)
- Clustering = MessageModel(2)
-)
-
-func (mode MessageModel) String() string {
- switch mode {
- case BroadCasting:
- return "BroadCasting"
- case Clustering:
- return "Clustering"
- default:
- return "Unknown"
- }
-}
-
-//ConsumerModel CoCurrently or Orderly
-type ConsumerModel int
-
-//ConsumerModel
-const (
- CoCurrently = ConsumerModel(1)
- Orderly = ConsumerModel(2)
-)
-
-func (mode ConsumerModel) String() string {
- switch mode {
- case CoCurrently:
- return "CoCurrently"
- case Orderly:
- return "Orderly"
- default:
- return "Unknown"
- }
-}
-
-// PushConsumerConfig define a new consumer.
-type PushConsumerConfig struct {
- ClientConfig
- ThreadCount int
- MessageBatchMaxSize int
- Model MessageModel
- ConsumerModel ConsumerModel
- MaxCacheMessageSize int
- MaxCacheMessageSizeInMB int
-}
-
-func (config *PushConsumerConfig) String() string {
- // For security, don't print Credentials.
- str := "PushConsumerConfig=[" + config.ClientConfig.String()
-
- if config.ThreadCount > 0 {
- str = strJoin(str, "ThreadCount", config.ThreadCount)
- }
-
- if config.MessageBatchMaxSize > 0 {
- str = strJoin(str, "MessageBatchMaxSize", config.MessageBatchMaxSize)
- }
-
- if config.Model != 0 {
- str = strJoin(str, "MessageModel", config.Model.String())
- }
-
- if config.ConsumerModel != 0 {
- str = strJoin(str, "ConsumerModel", config.ConsumerModel.String())
- }
-
- if config.MaxCacheMessageSize != 0 {
- str = strJoin(str, "MaxCacheMessageSize", config.MaxCacheMessageSize)
- }
-
- if config.MaxCacheMessageSizeInMB != 0 {
- str = strJoin(str, "MaxCacheMessageSizeInMB", config.MaxCacheMessageSizeInMB)
- }
- return str + "]"
-}
-
-// PushConsumer apis for PushConsumer
-type PushConsumer interface {
- baseAPI
-
- // Subscribe a new topic with specify filter expression and consume function.
- Subscribe(topic, expression string, consumeFunc func(msg *MessageExt) ConsumeStatus) error
-}
-
-// PullConsumerConfig the configuration for the pull consumer
-type PullConsumerConfig struct {
- ClientConfig
-}
-
-func (config *PullConsumerConfig) String() string {
- return "PushConsumerConfig=[" + config.ClientConfig.String() + "]"
-}
-
-// PullConsumer consumer pulling the message
-type PullConsumer interface {
- baseAPI
-
- // Pull returns the messages from the consume queue by specify the offset and the max number
- Pull(mq MessageQueue, subExpression string, offset int64, maxNums int) PullResult
-
- // FetchSubscriptionMessageQueues returns the consume queue of the topic
- FetchSubscriptionMessageQueues(topic string) []MessageQueue
-}
-
-//SessionCredentials access config for client
-type SessionCredentials struct {
- AccessKey string
- SecretKey string
- Channel string
-}
-
-func (session *SessionCredentials) String() string {
- return fmt.Sprintf("[accessKey: %s, secretKey: %s, channel: %s]",
- session.AccessKey, session.SecretKey, session.Channel)
-}
-
-//SendResult status for send
-type SendResult struct {
- Status SendStatus
- MsgId string
- Offset int64
-}
-
-func (result *SendResult) String() string {
- return fmt.Sprintf("[status: %s, messageId: %s, offset: %d]", result.Status, result.MsgId, result.Offset)
-}
-
-type baseAPI interface {
- Start() error
- Shutdown() error
-}
diff --git a/core/api_test.go b/core/api_test.go
deleted file mode 100644
index df7cfc0..0000000
--- a/core/api_test.go
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestProducerConfig_String(t *testing.T) {
- pConfig := ProducerConfig{}
- pConfig.GroupID = "testGroup"
- pConfig.NameServer = "localhost:9876"
- pConfig.NameServerDomain = "domain1"
- pConfig.InstanceName = "testProducer"
- pConfig.LogC = &LogConfig{
- Path: "/rocketmq/log",
- FileNum: 16,
- FileSize: 1 << 20,
- Level: LogLevelDebug}
- pConfig.SendMsgTimeout = 30
- pConfig.CompressLevel = 4
- pConfig.MaxMessageSize = 1024
- pConfig.ProducerModel = CommonProducer
-
- expect := "ProducerConfig=[GroupId: testGroup, NameServer: localhost:9876, NameServerDomain: domain1, InstanceName: testProducer, " +
- "LogConfig: {Path:/rocketmq/log FileNum:16 FileSize:1048576 Level:Debug}, " +
- "SendMsgTimeout: 30, CompressLevel: 4, MaxMessageSize: 1024, ProducerModel: CommonProducer, ]"
- assert.Equal(t, expect, pConfig.String())
-}
-
-func TestPushConsumerConfig_String(t *testing.T) {
- pcConfig := PushConsumerConfig{}
- pcConfig.GroupID = "testGroup"
- pcConfig.NameServer = "localhost:9876"
- pcConfig.InstanceName = "testPushConsumer"
- pcConfig.LogC = &LogConfig{
- Path: "/rocketmq/log",
- FileNum: 16,
- FileSize: 1 << 20,
- Level: LogLevelDebug}
- pcConfig.ThreadCount = 4
- pcConfig.MessageBatchMaxSize = 1024
- expect := "PushConsumerConfig=[GroupId: testGroup, NameServer: localhost:9876, " +
- "InstanceName: testPushConsumer, LogConfig: {Path:/rocketmq/log FileNum:16 FileSize:1048576 Level:Debug}, " +
- "ThreadCount: 4, MessageBatchMaxSize: 1024, ]"
- assert.Equal(t, expect, pcConfig.String())
-
- pcConfig.NameServerDomain = "domain1"
- expect = "PushConsumerConfig=[GroupId: testGroup, NameServer: localhost:9876, NameServerDomain: domain1, InstanceName: testPushConsumer, " +
- "LogConfig: {Path:/rocketmq/log FileNum:16 FileSize:1048576 Level:Debug}, ThreadCount: 4, MessageBatchMaxSize: 1024, ]"
- assert.Equal(t, expect, pcConfig.String())
-
- pcConfig.MessageBatchMaxSize = 32
- pcConfig.Model = Clustering
- pcConfig.ConsumerModel = CoCurrently
- pcConfig.MaxCacheMessageSize = 1024
- pcConfig.MaxCacheMessageSizeInMB = 2048
- expect = "PushConsumerConfig=[GroupId: testGroup, NameServer: localhost:9876, NameServerDomain: domain1, InstanceName: testPushConsumer, " +
- "LogConfig: {Path:/rocketmq/log FileNum:16 FileSize:1048576 Level:Debug}, ThreadCount: 4," +
- " MessageBatchMaxSize: 32, MessageModel: Clustering, ConsumerModel: CoCurrently," +
- " MaxCacheMessageSize: 1024, MaxCacheMessageSizeInMB: 2048, ]"
- assert.Equal(t, expect, pcConfig.String())
-}
-
-func TestPullConfig_String(t *testing.T) {
- pConfig := PullConsumerConfig{}
- pConfig.GroupID = "testGroup"
- pConfig.NameServer = "localhost:9876"
- pConfig.NameServerDomain = "domain1"
- pConfig.InstanceName = "testProducer"
- pConfig.LogC = &LogConfig{
- Path: "/rocketmq/log",
- FileNum: 16,
- FileSize: 1 << 20,
- Level: LogLevelDebug}
-
- expect := "PushConsumerConfig=[GroupId: testGroup, NameServer: localhost:9876, NameServerDomain: domain1, InstanceName: testProducer, " +
- "LogConfig: {Path:/rocketmq/log FileNum:16 FileSize:1048576 Level:Debug}, ]"
- assert.Equal(t, expect, pConfig.String())
-}
-
-func TestSessionCredentials_String(t *testing.T) {
- pConfig := SessionCredentials{}
- pConfig.AccessKey = "AK"
- pConfig.SecretKey = "SK"
- pConfig.Channel = "Cloud"
-
- expect := "[accessKey: AK, secretKey: SK, channel: Cloud]"
- assert.Equal(t, expect, pConfig.String())
-}
-
-func TestSendResult_String(t *testing.T) {
- pConfig := SendResult{}
- pConfig.Status = SendOK
- pConfig.MsgId = "MessageId"
- pConfig.Offset = 100000
-
- expect := "[status: SendOK, messageId: MessageId, offset: 100000]"
- assert.Equal(t, expect, pConfig.String())
-
- pConfig.Status = SendFlushDiskTimeout
- expect = "[status: SendFlushDiskTimeout, messageId: MessageId, offset: 100000]"
- assert.Equal(t, expect, pConfig.String())
-}
diff --git a/core/consumer_callback.go b/core/consumer_callback.go
deleted file mode 100644
index 4f17534..0000000
--- a/core/consumer_callback.go
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#cgo LDFLAGS: -L/usr/local/lib -lrocketmq
-
-#include <rocketmq/CMessageExt.h>
-#include <rocketmq/CPushConsumer.h>
-*/
-import "C"
-import (
- "sync"
-)
-
-var pushConsumerMap sync.Map
-
-//export consumeMessageCallback
-func consumeMessageCallback(cconsumer *C.CPushConsumer, msg *C.CMessageExt) C.int {
- consumer, exist := pushConsumerMap.Load(cconsumer)
- if !exist {
- return C.int(ReConsumeLater)
- }
-
- msgExt := cmsgExtToGo(msg)
- //C.DestroyMessageExt(msg)
- cfunc, exist := consumer.(*defaultPushConsumer).funcsMap.Load(msgExt.Topic)
- if !exist {
- return C.int(ReConsumeLater)
- }
- return C.int(cfunc.(func(msg *MessageExt) ConsumeStatus)(msgExt))
-}
diff --git a/core/error.go b/core/error.go
deleted file mode 100644
index 0f15a1e..0000000
--- a/core/error.go
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#include <rocketmq/CCommon.h>
-#include <rocketmq/CErrorMessage.h>
-*/
-import "C"
-import "fmt"
-
-type rmqError int
-
-//This is error messages
-const (
- NIL = rmqError(C.OK)
- ErrNullPoint = rmqError(C.NULL_POINTER)
- ErrMallocFailed = rmqError(C.MALLOC_FAILED)
- ErrProducerStartFailed = rmqError(C.PRODUCER_START_FAILED)
- ErrSendSyncFailed = rmqError(C.PRODUCER_SEND_SYNC_FAILED)
- ErrSendOnewayFailed = rmqError(C.PRODUCER_SEND_ONEWAY_FAILED)
- ErrSendOrderlyFailed = rmqError(C.PRODUCER_SEND_ORDERLY_FAILED)
- ErrSendTransactionFailed = rmqError(C.PRODUCER_SEND_TRANSACTION_FAILED)
- ErrPushConsumerStartFailed = rmqError(C.PUSHCONSUMER_START_FAILED)
- ErrPullConsumerStartFailed = rmqError(C.PULLCONSUMER_START_FAILED)
- ErrFetchMQFailed = rmqError(C.PULLCONSUMER_FETCH_MQ_FAILED)
- ErrFetchMessageFailed = rmqError(C.PULLCONSUMER_FETCH_MESSAGE_FAILED)
- ErrNotSupportNow = rmqError(C.NOT_SUPPORT_NOW)
-)
-
-func (e rmqError) Error() string {
- switch e {
- case ErrNullPoint:
- return "null point"
- case ErrMallocFailed:
- return "malloc memory failed"
- case ErrProducerStartFailed:
- return "start producer failed"
- case ErrSendSyncFailed:
- return "send message with sync failed"
- case ErrSendOrderlyFailed:
- return "send message with orderly failed"
- case ErrSendOnewayFailed:
- return "send message with oneway failed"
- case ErrSendTransactionFailed:
- return "send transaction message failed"
- case ErrPushConsumerStartFailed:
- return "start push-consumer failed"
- case ErrPullConsumerStartFailed:
- return "start pull-consumer failed"
- case ErrFetchMQFailed:
- return "fetch MessageQueue failed"
- case ErrFetchMessageFailed:
- return "fetch Message failed"
- case ErrNotSupportNow:
- return "this function is not support"
- default:
- return fmt.Sprintf("unknow error: %v", int(e))
- }
-}
-
-// GetLatestErrorMessage Get latest detailed error message from CPP-SDK
-func GetLatestErrorMessage() string {
- return C.GoString(C.GetLatestErrorMessage())
-}
\ No newline at end of file
diff --git a/core/error_test.go b/core/error_test.go
deleted file mode 100644
index a729aad..0000000
--- a/core/error_test.go
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestRMQError_OK(t *testing.T) {
- err := rmqError(0)
- assert.Equal(t, NIL, err)
-}
-func TestRMQError_Failed(t *testing.T) {
- err := rmqError(int(ErrNullPoint))
- assert.Equal(t, ErrNullPoint, err)
-}
-func TestRMQError_String(t *testing.T) {
- err := rmqError(1)
- expect := "null point"
- assert.Equal(t, expect, err.Error())
-}
-
-func TestRMQError_Unknown(t *testing.T) {
- err := rmqError(-1000)
- expect := "unknow error: -1000"
- assert.Equal(t, expect, err.Error())
-}
-
-func TestRMQError_ErrorCode(t *testing.T) {
- assert.Equal(t, "malloc memory failed", rmqError(int(ErrMallocFailed)).Error())
- assert.Equal(t, "start producer failed", rmqError(int(ErrProducerStartFailed)).Error())
- assert.Equal(t, "send message with sync failed", rmqError(int(ErrSendSyncFailed)).Error())
- assert.Equal(t, "send message with orderly failed", rmqError(int(ErrSendOrderlyFailed)).Error())
- assert.Equal(t, "send message with oneway failed", rmqError(int(ErrSendOnewayFailed)).Error())
- assert.Equal(t, "send transaction message failed", rmqError(int(ErrSendTransactionFailed)).Error())
- assert.Equal(t, "start push-consumer failed", rmqError(int(ErrPushConsumerStartFailed)).Error())
- assert.Equal(t, "start pull-consumer failed", rmqError(int(ErrPullConsumerStartFailed)).Error())
- assert.Equal(t, "fetch MessageQueue failed", rmqError(int(ErrFetchMQFailed)).Error())
- assert.Equal(t, "fetch Message failed", rmqError(int(ErrFetchMessageFailed)).Error())
- assert.Equal(t, "this function is not support", rmqError(int(ErrNotSupportNow)).Error())
-}
diff --git a/core/log.go b/core/log.go
deleted file mode 100644
index 18a5275..0000000
--- a/core/log.go
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#include <rocketmq/CCommon.h>
-*/
-import "C"
-import "fmt"
-
-// LogLevel the log level
-type LogLevel int
-
-// predefined log level
-const (
- LogLevelFatal = LogLevel(C.E_LOG_LEVEL_FATAL)
- LogLevelError = LogLevel(C.E_LOG_LEVEL_ERROR)
- LogLevelWarn = LogLevel(C.E_LOG_LEVEL_WARN)
- LogLevelInfo = LogLevel(C.E_LOG_LEVEL_INFO)
- LogLevelDebug = LogLevel(C.E_LOG_LEVEL_DEBUG)
- LogLevelTrace = LogLevel(C.E_LOG_LEVEL_TRACE)
- LogLevelNum = LogLevel(C.E_LOG_LEVEL_LEVEL_NUM)
-)
-
-func (l LogLevel) String() string {
- switch l {
- case LogLevelFatal:
- return "Fatal"
- case LogLevelError:
- return "Error"
- case LogLevelWarn:
- return "Warn"
- case LogLevelInfo:
- return "Info"
- case LogLevelDebug:
- return "Debug"
- case LogLevelTrace:
- return "Trace"
- case LogLevelNum:
- return "Num"
- default:
- return "Unkonw"
- }
-}
-
-// LogConfig the log configuration for the pull consumer
-type LogConfig struct {
- Path string
- FileNum int
- FileSize int64
- Level LogLevel
-}
-
-func (lc *LogConfig) String() string {
- return fmt.Sprintf("%+v", *lc)
-}
diff --git a/core/log_test.go b/core/log_test.go
deleted file mode 100644
index 86cf4c5..0000000
--- a/core/log_test.go
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestLogConfig_String(t *testing.T) {
- logc := LogConfig{Path: "/log/path1", FileNum: 3, FileSize: 1 << 20, Level: LogLevelDebug}
- assert.Equal(t, "{Path:/log/path1 FileNum:3 FileSize:1048576 Level:Debug}", logc.String())
- logc.Level = LogLevelFatal
- assert.Equal(t, "{Path:/log/path1 FileNum:3 FileSize:1048576 Level:Fatal}", logc.String())
- logc.Level = LogLevelError
- assert.Equal(t, "{Path:/log/path1 FileNum:3 FileSize:1048576 Level:Error}", logc.String())
- logc.Level = LogLevelWarn
- assert.Equal(t, "{Path:/log/path1 FileNum:3 FileSize:1048576 Level:Warn}", logc.String())
- logc.Level = LogLevelInfo
- assert.Equal(t, "{Path:/log/path1 FileNum:3 FileSize:1048576 Level:Info}", logc.String())
- logc.Level = LogLevelTrace
- assert.Equal(t, "{Path:/log/path1 FileNum:3 FileSize:1048576 Level:Trace}", logc.String())
- logc.Level = LogLevelError
-}
-func TestLogLevel_String(t *testing.T) {
- logc := LogConfig{Path: "/log/path1", FileNum: 3, FileSize: 1 << 20, Level: LogLevelDebug}
- assert.Equal(t, "Debug", logc.Level.String())
-}
diff --git a/core/message.go b/core/message.go
deleted file mode 100644
index 83db646..0000000
--- a/core/message.go
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#cgo LDFLAGS: -L/usr/local/lib/ -lrocketmq
-#include <rocketmq/CMessage.h>
-#include <rocketmq/CMessageExt.h>
-#include <stdlib.h>
-*/
-import "C"
-import (
- "fmt"
- "unsafe"
-)
-
-//Message used for send
-type Message struct {
- Topic string
- Tags string
- Keys string
- Body string
- DelayTimeLevel int
- Property map[string]string
- cmsg *C.struct_CMessage
-}
-
-func (msg *Message) String() string {
- return fmt.Sprintf("[Topic: %s, Tags: %s, Keys: %s, Body: %s, DelayTimeLevel: %d, Property: %v]",
- msg.Topic, msg.Tags, msg.Keys, msg.Body, msg.DelayTimeLevel, msg.Property)
-}
-
-//GetProperty get message property by key string
-func (msg *Message) GetProperty(key string) string {
- ck := C.CString(key)
- defer C.free(unsafe.Pointer(ck))
- return C.GoString(C.GetOriginMessageProperty(msg.cmsg, ck))
-}
-
-func goMsgToC(gomsg *Message) *C.struct_CMessage {
- cs := C.CString(gomsg.Topic)
- var cmsg = C.CreateMessage(cs)
- C.free(unsafe.Pointer(cs))
-
- cs = C.CString(gomsg.Tags)
- C.SetMessageTags(cmsg, cs)
- C.free(unsafe.Pointer(cs))
-
- cs = C.CString(gomsg.Keys)
- C.SetMessageKeys(cmsg, cs)
- C.free(unsafe.Pointer(cs))
-
- cs = C.CString(gomsg.Body)
- C.SetMessageBody(cmsg, cs)
- C.free(unsafe.Pointer(cs))
-
- C.SetDelayTimeLevel(cmsg, C.int(gomsg.DelayTimeLevel))
-
- for k, v := range gomsg.Property {
- key := C.CString(k)
- value := C.CString(v)
- C.SetMessageProperty(cmsg, key, value)
- C.free(unsafe.Pointer(key))
- C.free(unsafe.Pointer(value))
- }
- return cmsg
-}
-
-func cMsgToGo(cMsg *C.struct_CMessage) *Message {
- gomsg := &Message{}
-
- gomsg.Topic = C.GoString(C.GetOriginMessageTopic(cMsg))
- gomsg.Tags = C.GoString(C.GetOriginMessageTags(cMsg))
- gomsg.Keys = C.GoString(C.GetOriginMessageKeys(cMsg))
- gomsg.Body = C.GoString(C.GetOriginMessageBody(cMsg))
- gomsg.DelayTimeLevel = int(C.GetOriginDelayTimeLevel(cMsg))
- gomsg.cmsg = cMsg
-
- return gomsg
-}
-
-//MessageExt used for consume
-type MessageExt struct {
- Message
- MessageID string
- QueueId int
- ReconsumeTimes int
- StoreSize int
- BornTimestamp int64
- StoreTimestamp int64
- QueueOffset int64
- CommitLogOffset int64
- PreparedTransactionOffset int64
-
- // improve: is there is a method convert c++ map to go variable?
- cmsgExt *C.struct_CMessageExt
-}
-
-func (msgExt *MessageExt) String() string {
- return fmt.Sprintf("[MessageId: %s, %s, QueueId: %d, ReconsumeTimes: %d, StoreSize: %d, BornTimestamp: %d, "+
- "StoreTimestamp: %d, QueueOffset: %d, CommitLogOffset: %d, PreparedTransactionOffset: %d]", msgExt.MessageID,
- msgExt.Message.String(), msgExt.QueueId, msgExt.ReconsumeTimes, msgExt.StoreSize, msgExt.BornTimestamp,
- msgExt.StoreTimestamp, msgExt.QueueOffset, msgExt.CommitLogOffset, msgExt.PreparedTransactionOffset)
-}
-
-//GetProperty get the message property by key from message ext
-func (msgExt *MessageExt) GetProperty(key string) string {
- ck := C.CString(key)
- defer C.free(unsafe.Pointer(ck))
- return C.GoString(C.GetMessageProperty(msgExt.cmsgExt, ck))
-}
-
-func cmsgExtToGo(cmsg *C.struct_CMessageExt) *MessageExt {
- gomsg := &MessageExt{cmsgExt: cmsg}
-
- gomsg.Topic = C.GoString(C.GetMessageTopic(cmsg))
- gomsg.Tags = C.GoString(C.GetMessageTags(cmsg))
- gomsg.Keys = C.GoString(C.GetMessageKeys(cmsg))
- gomsg.Body = C.GoString(C.GetMessageBody(cmsg))
- gomsg.MessageID = C.GoString(C.GetMessageId(cmsg))
- gomsg.DelayTimeLevel = int(C.GetMessageDelayTimeLevel(cmsg))
- gomsg.QueueId = int(C.GetMessageQueueId(cmsg))
- gomsg.ReconsumeTimes = int(C.GetMessageReconsumeTimes(cmsg))
- gomsg.StoreSize = int(C.GetMessageStoreSize(cmsg))
- gomsg.BornTimestamp = int64(C.GetMessageBornTimestamp(cmsg))
- gomsg.StoreTimestamp = int64(C.GetMessageStoreTimestamp(cmsg))
- gomsg.QueueOffset = int64(C.GetMessageQueueOffset(cmsg))
- gomsg.CommitLogOffset = int64(C.GetMessageCommitLogOffset(cmsg))
- gomsg.PreparedTransactionOffset = int64(C.GetMessagePreparedTransactionOffset(cmsg))
-
- return gomsg
-}
diff --git a/core/message_test.go b/core/message_test.go
deleted file mode 100644
index c4183a4..0000000
--- a/core/message_test.go
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestMessage_String(t *testing.T) {
- msg := Message{
- Topic: "testTopic",
- Tags: "TagA, TagB",
- Keys: "Key1, Key2",
- Body: "Body1234567890",
- DelayTimeLevel: 8}
- expect := "[Topic: testTopic, Tags: TagA, TagB, Keys: Key1, Key2, Body: Body1234567890, DelayTimeLevel: 8," +
- " Property: map[]]"
- assert.Equal(t, expect, msg.String())
-}
-
-func TestMessage_GetProperty(t *testing.T) {
- msg := Message{
- Topic: "testTopic",
- Tags: "TagA, TagB",
- Keys: "Key1, Key2",
- Body: "Body1234567890",
- DelayTimeLevel: 8}
- cmsg := goMsgToC(&msg)
- newMsg := cMsgToGo(cmsg)
- expect := "[Topic: testTopic, Tags: TagA, TagB, Keys: Key1, Key2, Body: Body1234567890, DelayTimeLevel: 8," +
- " Property: map[]]"
- assert.Equal(t, expect, newMsg.String())
- val := newMsg.GetProperty("KEY")
- assert.Empty(t, val)
-}
-
-func TestMessageExt_String(t *testing.T) {
- msg := Message{
- Topic: "testTopic",
- Tags: "TagA, TagB",
- Keys: "Key1, Key2",
- Body: "Body1234567890",
- DelayTimeLevel: 8}
- msgExt := MessageExt{
- Message: msg,
- MessageID: "messageId",
- QueueId: 2,
- ReconsumeTimes: 13,
- StoreSize: 1 << 10,
- BornTimestamp: int64(1234567890897),
- StoreTimestamp: int64(1234567890),
- QueueOffset: int64(1234567890),
- CommitLogOffset: int64(1234567890),
- PreparedTransactionOffset: int64(1234567890),
- }
- expect := "[MessageId: messageId, [Topic: testTopic, Tags: TagA, TagB, Keys: Key1, Key2, " +
- "Body: Body1234567890, DelayTimeLevel: 8, Property: map[]], QueueId: 2, ReconsumeTimes: " +
- "13, StoreSize: 1024, BornTimestamp: 1234567890897, StoreTimestamp: 1234567890, QueueOffset: 1234567890," +
- " CommitLogOffset: 1234567890, PreparedTransactionOffset: 1234567890]"
- assert.Equal(t, expect, msgExt.String())
-}
diff --git a/core/producer.go b/core/producer.go
deleted file mode 100644
index e28b652..0000000
--- a/core/producer.go
+++ /dev/null
@@ -1,309 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#cgo LDFLAGS: -L/usr/local/lib/ -lrocketmq
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <rocketmq/CMessage.h>
-#include <rocketmq/CProducer.h>
-#include <rocketmq/CSendResult.h>
-
-int queueSelectorCallback_cgo(int size, CMessage *msg, void *selectorKey) {
- int queueSelectorCallback(int, void*);
- return queueSelectorCallback(size, selectorKey);
-}
-*/
-import "C"
-import (
- "errors"
- log "github.com/sirupsen/logrus"
- "unsafe"
-)
-
-//SendStatus The Status for send result from C apis.
-type SendStatus int
-
-const (
- //SendOK OK
- SendOK = SendStatus(C.E_SEND_OK)
- //SendFlushDiskTimeout Failed because broker flush error
- SendFlushDiskTimeout = SendStatus(C.E_SEND_FLUSH_DISK_TIMEOUT)
- //SendFlushSlaveTimeout Failed because slave broker timeout
- SendFlushSlaveTimeout = SendStatus(C.E_SEND_FLUSH_SLAVE_TIMEOUT)
- //SendSlaveNotAvailable Failed because slave broker error
- SendSlaveNotAvailable = SendStatus(C.E_SEND_SLAVE_NOT_AVAILABLE)
-)
-
-func (status SendStatus) String() string {
- switch status {
- case SendOK:
- return "SendOK"
- case SendFlushDiskTimeout:
- return "SendFlushDiskTimeout"
- case SendFlushSlaveTimeout:
- return "SendFlushSlaveTimeout"
- case SendSlaveNotAvailable:
- return "SendSlaveNotAvailable"
- default:
- return "Unknown"
- }
-}
-
-func newDefaultProducer(config *ProducerConfig) (*defaultProducer, error) {
- if config == nil {
- return nil, errors.New("config is nil")
- }
-
- if config.GroupID == "" {
- return nil, errors.New("GroupId is empty")
- }
-
- if config.NameServer == "" && config.NameServerDomain == "" {
- return nil, errors.New("NameServer and NameServerDomain is empty")
- }
-
- producer := &defaultProducer{config: config}
- cs := C.CString(config.GroupID)
- var cproducer *C.struct_CProducer
- if config.ProducerModel == OrderlyProducer {
- cproducer = C.CreateOrderlyProducer(cs)
- } else if config.ProducerModel == CommonProducer {
- cproducer = C.CreateProducer(cs)
- } else {
- C.free(unsafe.Pointer(cs))
- return nil, errors.New("ProducerModel is invalid or empty")
- }
- C.free(unsafe.Pointer(cs))
-
- if cproducer == nil {
- return nil, errors.New("create Producer failed")
- }
-
- var err rmqError
- if config.NameServer != "" {
- cs = C.CString(config.NameServer)
- err = rmqError(C.SetProducerNameServerAddress(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.NameServerDomain != "" {
- cs = C.CString(config.NameServerDomain)
- err = rmqError(C.SetProducerNameServerDomain(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.InstanceName != "" {
- cs = C.CString(config.InstanceName)
- err = rmqError(C.SetProducerInstanceName(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.Credentials != nil {
- ak := C.CString(config.Credentials.AccessKey)
- sk := C.CString(config.Credentials.SecretKey)
- ch := C.CString(config.Credentials.Channel)
- err = rmqError(C.SetProducerSessionCredentials(cproducer, ak, sk, ch))
-
- C.free(unsafe.Pointer(ak))
- C.free(unsafe.Pointer(sk))
- C.free(unsafe.Pointer(ch))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.LogC != nil {
- cs = C.CString(config.LogC.Path)
- err = rmqError(C.SetProducerLogPath(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetProducerLogFileNumAndSize(cproducer, C.int(config.LogC.FileNum), C.long(config.LogC.FileSize)))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetProducerLogLevel(cproducer, C.CLogLevel(config.LogC.Level)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.SendMsgTimeout > 0 {
- err = rmqError(C.SetProducerSendMsgTimeout(cproducer, C.int(config.SendMsgTimeout)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.CompressLevel > 0 {
- err = rmqError(C.SetProducerCompressLevel(cproducer, C.int(config.CompressLevel)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.MaxMessageSize > 0 {
- err = rmqError(C.SetProducerMaxMessageSize(cproducer, C.int(config.MaxMessageSize)))
- if err != NIL {
- return nil, err
- }
- }
-
- producer.cproducer = cproducer
- return producer, nil
-}
-
-type defaultProducer struct {
- config *ProducerConfig
- cproducer *C.struct_CProducer
-}
-
-func (p *defaultProducer) String() string {
- return p.config.String()
-}
-
-// Start the producer.
-func (p *defaultProducer) Start() error {
- err := rmqError(C.StartProducer(p.cproducer))
- if err != NIL {
- return err
- }
- return nil
-}
-
-// Shutdown the producer.
-func (p *defaultProducer) Shutdown() error {
- err := rmqError(C.ShutdownProducer(p.cproducer))
-
- if err != NIL {
- return err
- }
-
- err = rmqError(int(C.DestroyProducer(p.cproducer)))
- if err != NIL {
- return err
- }
-
- return err
-}
-
-func (p *defaultProducer) SendMessageSync(msg *Message) (*SendResult, error) {
- cmsg := goMsgToC(msg)
- defer C.DestroyMessage(cmsg)
-
- var sr C.struct__SendResult_
- err := rmqError(C.SendMessageSync(p.cproducer, cmsg, &sr))
-
- if err != NIL {
- log.Warnf("send message error, error is: %s", err.Error())
- return nil, err
- }
-
- result := &SendResult{}
- result.Status = SendStatus(sr.sendStatus)
- result.MsgId = C.GoString(&sr.msgId[0])
- result.Offset = int64(sr.offset)
- return result, nil
-}
-
-func (p *defaultProducer) SendMessageOrderly(msg *Message, selector MessageQueueSelector, arg interface{}, autoRetryTimes int) (*SendResult, error) {
- if p.config.ProducerModel == OrderlyProducer {
- log.Warnf("Can not send message orderly by common select queue in lite order producer")
- return nil, ErrSendOrderlyFailed
- }
- cmsg := goMsgToC(msg)
- defer C.DestroyMessage(cmsg)
- key := selectors.put(&messageQueueSelectorWrapper{selector: selector, m: msg, arg: arg})
-
- var sr C.struct__SendResult_
- err := rmqError(C.SendMessageOrderly(
- p.cproducer,
- cmsg,
- (C.QueueSelectorCallback)(unsafe.Pointer(C.queueSelectorCallback_cgo)),
- unsafe.Pointer(&key),
- C.int(autoRetryTimes),
- &sr))
-
- if err != NIL {
- log.Warnf("send message orderly error, error is: %s", err.Error())
- return nil, err
- }
-
- return &SendResult{
- Status: SendStatus(sr.sendStatus),
- MsgId: C.GoString(&sr.msgId[0]),
- Offset: int64(sr.offset),
- }, nil
-}
-
-func (p *defaultProducer) SendMessageOneway(msg *Message) error {
- cmsg := goMsgToC(msg)
- defer C.DestroyMessage(cmsg)
-
- err := rmqError(C.SendMessageOneway(p.cproducer, cmsg))
- if err != NIL {
- log.Warnf("send message with oneway error, error is: %s", err.Error())
- return err
- }
-
- log.Debugf("Send Message: %s with oneway success.", msg.String())
- return nil
-}
-
-func (p *defaultProducer) SendMessageOrderlyByShardingKey(msg *Message, shardingkey string) (*SendResult, error) {
- if p.config.ProducerModel != OrderlyProducer {
- log.Warnf("Can not send message orderly, This method only support in lite order producer.")
- return nil, ErrSendOrderlyFailed
- }
- cmsg := goMsgToC(msg)
- defer C.DestroyMessage(cmsg)
- cshardingkey := C.CString(shardingkey)
- defer C.free(unsafe.Pointer(cshardingkey))
- var sr C.struct__SendResult_
- err := rmqError(C.SendMessageOrderlyByShardingKey(
- p.cproducer,
- cmsg,
- cshardingkey,
- &sr))
-
- if err != NIL {
- log.Warnf("send message orderly error, error is: %s", err.Error())
- return nil, err
- }
-
- return &SendResult{
- Status: SendStatus(sr.sendStatus),
- MsgId: C.GoString(&sr.msgId[0]),
- Offset: int64(sr.offset),
- }, nil
-}
diff --git a/core/producer_test.go b/core/producer_test.go
deleted file mode 100644
index 61cb3c6..0000000
--- a/core/producer_test.go
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "errors"
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestProducer_SendStatus(t *testing.T) {
- assert.Equal(t, "SendOK", SendStatus(int(SendOK)).String())
- assert.Equal(t, "SendFlushDiskTimeout", SendStatus(int(SendFlushDiskTimeout)).String())
- assert.Equal(t, "SendFlushSlaveTimeout", SendStatus(int(SendFlushSlaveTimeout)).String())
- assert.Equal(t, "SendSlaveNotAvailable", SendStatus(int(SendSlaveNotAvailable)).String())
- assert.Equal(t, "Unknown", SendStatus(int(-1)).String())
-}
-
-func TestProducer_CreateProducerFailed(t *testing.T) {
- pConfig := ProducerConfig{}
-
- producer, err := newDefaultProducer(nil)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("config is nil"))
- producer, err = newDefaultProducer(&pConfig)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("GroupId is empty"))
- pConfig.GroupID = "testGroupA"
- producer, err = newDefaultProducer(&pConfig)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("NameServer and NameServerDomain is empty"))
- pConfig.NameServer = "localhost:9876"
- pConfig.ProducerModel = TransProducer
- producer, err = newDefaultProducer(&pConfig)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("ProducerModel is invalid or empty"))
-}
-
-func TestProducer_CreateProducer(t *testing.T) {
- pConfig := ProducerConfig{}
- pConfig.GroupID = "testGroupB"
- pConfig.NameServer = "localhost:9876"
- pConfig.InstanceName = "testProducer"
- pConfig.Credentials = &SessionCredentials{
- AccessKey: "AK",
- SecretKey: "SK",
- Channel: "Cloud"}
- pConfig.LogC = &LogConfig{
- Path: "/rocketmq/log",
- FileNum: 16,
- FileSize: 1 << 20,
- Level: LogLevelDebug}
- pConfig.SendMsgTimeout = 30
- pConfig.CompressLevel = 4
- pConfig.MaxMessageSize = 1024
- pConfig.ProducerModel = CommonProducer
-
- producer, err := newDefaultProducer(&pConfig)
- assert.Nil(t, err)
- assert.NotEmpty(t, producer)
-}
-
-func TestDefaultProducer_SendMessageSync(t *testing.T) {
- pConfig := ProducerConfig{}
- pConfig.GroupID = "testGroupSync"
- pConfig.NameServer = "localhost:9876"
- pConfig.ProducerModel = CommonProducer
-
- producer, err := newDefaultProducer(&pConfig)
- assert.Nil(t, err)
- assert.NotEmpty(t, producer)
- err = producer.Start()
- assert.Nil(t, err)
- msg := &Message{
- Topic: "test",
- Tags: "TagA",
- Keys: "Key",
- Body: "Body1234567890"}
- producer.SendMessageSync(msg)
- //sr, errors := producer.SendMessageSync(msg)
- //assert.Nil(t, errors)
- //assert.NotEmpty(t, sr.MsgId)
- //producer.Shutdown()
-}
-
-func TestDefaultProducer_SendMessageOneway(t *testing.T) {
- pConfig := ProducerConfig{}
- pConfig.GroupID = "testGroupOneway"
- pConfig.NameServer = "localhost:9876"
- pConfig.ProducerModel = CommonProducer
-
- producer, err := newDefaultProducer(&pConfig)
- assert.Nil(t, err)
- assert.NotEmpty(t, producer)
- err = producer.Start()
- assert.Nil(t, err)
- msg := &Message{
- Topic: "test",
- Tags: "TagA",
- Keys: "Key",
- Body: "Body1234567890"}
- producer.SendMessageOneway(msg)
- //errors := producer.SendMessageOneway(msg)
- //assert.Nil(t, errors)
- //producer.Shutdown()
-}
-
-func TestDefaultProducer_SendMessageOrderlyByShardingKey(t *testing.T) {
- pConfig := ProducerConfig{}
- pConfig.GroupID = "testGroupOrderlyByKey"
- pConfig.NameServer = "localhost:9876"
- pConfig.ProducerModel = OrderlyProducer
-
- producer, err := newDefaultProducer(&pConfig)
- assert.Nil(t, err)
- assert.NotEmpty(t, producer)
- err = producer.Start()
- assert.Nil(t, err)
- msg := &Message{
- Topic: "test",
- Tags: "TagA",
- Keys: "Key",
- Body: "Body1234567890"}
- producer.SendMessageOrderlyByShardingKey(msg, "key")
- //sr, errors := producer.SendMessageOrderlyByShardingKey(msg, "key")
- //assert.Nil(t, errors)
- //assert.NotEmpty(t, sr.MsgId)
- //producer.Shutdown()
-}
-
-type testMessageQueueSelector struct {
-}
-
-func (m *testMessageQueueSelector) Select(size int, msg *Message, arg interface{}) int {
- return 0
-}
-func TestDefaultProducer_SendMessageOrderly(t *testing.T) {
- pConfig := ProducerConfig{}
- pConfig.GroupID = "testGroupOrderly"
- pConfig.NameServer = "localhost:9876"
- pConfig.ProducerModel = OrderlyProducer
-
- producer, err := newDefaultProducer(&pConfig)
- assert.Nil(t, err)
- assert.NotEmpty(t, producer)
- err = producer.Start()
- assert.Nil(t, err)
- msg := &Message{
- Topic: "test",
- Tags: "TagA",
- Keys: "Key",
- Body: "Body1234567890"}
- s := &testMessageQueueSelector{}
- producer.SendMessageOrderly(msg, s, nil, 1)
- //sr, errors := producer.SendMessageOrderly(msg, s, nil, 1)
- //assert.Nil(t, errors)
- //assert.NotEmpty(t, sr.MsgId)
- //producer.Shutdown()
-}
diff --git a/core/pull_consumer.go b/core/pull_consumer.go
deleted file mode 100644
index 50b5196..0000000
--- a/core/pull_consumer.go
+++ /dev/null
@@ -1,251 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#cgo LDFLAGS: -L/usr/local/lib -lrocketmq
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <rocketmq/CMessageExt.h>
-#include <rocketmq/CPullConsumer.h>
-*/
-import "C"
-
-import (
- "errors"
- "fmt"
- "sync"
- "unsafe"
-)
-
-// PullStatus pull status
-type PullStatus int
-
-// predefined pull status
-const (
- PullFound = PullStatus(C.E_FOUND)
- PullNoNewMsg = PullStatus(C.E_NO_NEW_MSG)
- PullNoMatchedMsg = PullStatus(C.E_NO_MATCHED_MSG)
- PullOffsetIllegal = PullStatus(C.E_OFFSET_ILLEGAL)
- PullBrokerTimeout = PullStatus(C.E_BROKER_TIMEOUT)
-)
-
-func (ps PullStatus) String() string {
- switch ps {
- case PullFound:
- return "Found"
- case PullNoNewMsg:
- return "NoNewMsg"
- case PullNoMatchedMsg:
- return "NoMatchedMsg"
- case PullOffsetIllegal:
- return "OffsetIllegal"
- case PullBrokerTimeout:
- return "BrokerTimeout"
- default:
- return "Unknown status"
- }
-}
-
-// defaultPullConsumer default consumer pulling the message
-type defaultPullConsumer struct {
- PullConsumerConfig
- cconsumer *C.struct_CPullConsumer
- funcsMap sync.Map
-}
-
-func (c *defaultPullConsumer) String() string {
- topics := ""
- c.funcsMap.Range(func(key, value interface{}) bool {
- topics += key.(string) + ", "
- return true
- })
- return fmt.Sprintf("[%+v, subcribed topics: [%s]]", c.PullConsumerConfig, topics)
-}
-
-// NewPullConsumer creates a pull consumer
-func NewPullConsumer(config *PullConsumerConfig) (PullConsumer, error) {
- if config == nil {
- return nil, errors.New("config is nil")
- }
- if config.GroupID == "" {
- return nil, errors.New("GroupId is empty")
- }
-
- if config.NameServer == "" && config.NameServerDomain == "" {
- return nil, errors.New("NameServer and NameServerDomain is empty")
- }
-
- cs := C.CString(config.GroupID)
- cconsumer := C.CreatePullConsumer(cs)
- C.free(unsafe.Pointer(cs))
- if cconsumer == nil {
- return nil, errors.New("create PullConsumer failed")
- }
-
- var err rmqError
- if config.NameServer != "" {
- cs = C.CString(config.NameServer)
- err = rmqError(C.SetPullConsumerNameServerAddress(cconsumer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.NameServerDomain != "" {
- cs = C.CString(config.NameServerDomain)
- err = rmqError(C.SetPullConsumerNameServerDomain(cconsumer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.Credentials != nil {
- ak := C.CString(config.Credentials.AccessKey)
- sk := C.CString(config.Credentials.SecretKey)
- ch := C.CString(config.Credentials.Channel)
- err = rmqError(C.SetPullConsumerSessionCredentials(cconsumer, ak, sk, ch))
- C.free(unsafe.Pointer(ak))
- C.free(unsafe.Pointer(sk))
- C.free(unsafe.Pointer(ch))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.LogC != nil {
- cs = C.CString(config.LogC.Path)
- err = rmqError(C.SetPullConsumerLogPath(cconsumer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetPullConsumerLogFileNumAndSize(cconsumer, C.int(config.LogC.FileNum), C.long(config.LogC.FileSize)))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetPullConsumerLogLevel(cconsumer, C.CLogLevel(config.LogC.Level)))
- if err != NIL {
- return nil, err
- }
- }
-
- return &defaultPullConsumer{PullConsumerConfig: *config, cconsumer: cconsumer}, nil
-}
-
-// Start starts the pulling consumer
-func (c *defaultPullConsumer) Start() error {
- err := rmqError(C.StartPullConsumer(c.cconsumer))
- if err != NIL {
- return err
- }
- return nil
-}
-
-// Shutdown shutdown the pulling consumer
-func (c *defaultPullConsumer) Shutdown() error {
- err := rmqError(C.ShutdownPullConsumer(c.cconsumer))
- if err != NIL {
- return err
- }
-
- err = rmqError(C.DestroyPullConsumer(c.cconsumer))
- if err != NIL {
- return err
- }
- return nil
-}
-
-// FetchSubscriptionMessageQueues returns the topic's consume queue
-func (c *defaultPullConsumer) FetchSubscriptionMessageQueues(topic string) []MessageQueue {
- var (
- q *C.struct__CMessageQueue_
- size C.int
- )
-
- ctopic := C.CString(topic)
- C.FetchSubscriptionMessageQueues(c.cconsumer, ctopic, &q, &size)
- C.free(unsafe.Pointer(ctopic))
- if size == 0 {
- return nil
- }
-
- qs := make([]MessageQueue, size)
- for i := range qs {
- cq := (*C.struct__CMessageQueue_)(
- unsafe.Pointer(uintptr(unsafe.Pointer(q)) + uintptr(i)*unsafe.Sizeof(*q)),
- )
- qs[i].ID, qs[i].Broker, qs[i].Topic = int(cq.queueId), C.GoString(&cq.brokerName[0]), topic
- }
- C.ReleaseSubscriptionMessageQueue(q)
-
- return qs
-}
-
-// PullResult the pull result
-type PullResult struct {
- NextBeginOffset int64
- MinOffset int64
- MaxOffset int64
- Status PullStatus
- Messages []*MessageExt
-}
-
-func (pr *PullResult) String() string {
- return fmt.Sprintf("%+v", *pr)
-}
-
-// Pull pulling the message from the specified message queue
-func (c *defaultPullConsumer) Pull(mq MessageQueue, subExpression string, offset int64, maxNums int) PullResult {
- cmq := C.struct__CMessageQueue_{
- queueId: C.int(mq.ID),
- }
-
- copy(cmq.topic[:], *(*[]C.char)(unsafe.Pointer(&mq.Topic)))
- copy(cmq.brokerName[:], *(*[]C.char)(unsafe.Pointer(&mq.Broker)))
-
- csubExpr := C.CString(subExpression)
- cpullResult := C.Pull(c.cconsumer, &cmq, csubExpr, C.longlong(offset), C.int(maxNums))
-
- pr := PullResult{
- NextBeginOffset: int64(cpullResult.nextBeginOffset),
- MinOffset: int64(cpullResult.minOffset),
- MaxOffset: int64(cpullResult.maxOffset),
- Status: PullStatus(cpullResult.pullStatus),
- }
- if cpullResult.size > 0 {
- msgs := make([]*MessageExt, cpullResult.size)
- for i := range msgs {
- msgs[i] = cmsgExtToGo(*(**C.struct_CMessageExt)(
- unsafe.Pointer(
- uintptr(unsafe.Pointer(cpullResult.msgFoundList)) + uintptr(i)*unsafe.Sizeof(*cpullResult.msgFoundList),
- ),
- ))
- }
- pr.Messages = msgs
- }
-
- C.free(unsafe.Pointer(csubExpr))
- C.ReleasePullResult(cpullResult)
- return pr
-}
diff --git a/core/pull_consumer_test.go b/core/pull_consumer_test.go
deleted file mode 100644
index 528dc9c..0000000
--- a/core/pull_consumer_test.go
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "errors"
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestPullConsumer_PullStatus(t *testing.T) {
- assert.Equal(t, "Found", PullStatus(int(PullFound)).String())
- assert.Equal(t, "NoNewMsg", PullStatus(int(PullNoNewMsg)).String())
- assert.Equal(t, "NoMatchedMsg", PullStatus(int(PullNoMatchedMsg)).String())
- assert.Equal(t, "OffsetIllegal", PullStatus(int(PullOffsetIllegal)).String())
- assert.Equal(t, "BrokerTimeout", PullStatus(int(PullBrokerTimeout)).String())
- assert.Equal(t, "Unknown status", PullStatus(int(-1)).String())
-}
-
-func TestPullConsumer_CreatePullCosumerFailed(t *testing.T) {
- pConfig := PullConsumerConfig{}
-
- producer, err := NewPullConsumer(nil)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("config is nil"))
- producer, err = NewPullConsumer(&pConfig)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("GroupId is empty"))
- pConfig.GroupID = "testGroup"
- producer, err = NewPullConsumer(&pConfig)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("NameServer and NameServerDomain is empty"))
-}
diff --git a/core/push_consumer.go b/core/push_consumer.go
deleted file mode 100644
index 0dc27c4..0000000
--- a/core/push_consumer.go
+++ /dev/null
@@ -1,248 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#cgo LDFLAGS: -L/usr/local/lib -lrocketmq
-
-#include <stdlib.h>
-#include <rocketmq/CMessageExt.h>
-#include <rocketmq/CPushConsumer.h>
-
-extern int consumeMessageCallback(CPushConsumer *consumer, CMessageExt *msg);
-
-int callback_cgo(CPushConsumer *consumer, CMessageExt *msg) {
- return consumeMessageCallback(consumer, msg);
-}
-*/
-import "C"
-
-import (
- "errors"
- "fmt"
- log "github.com/sirupsen/logrus"
- "sync"
- "unsafe"
-)
-
-//ConsumeStatus the retern value for consumer
-type ConsumeStatus int
-
-const (
- //ConsumeSuccess commit offset to broker
- ConsumeSuccess = ConsumeStatus(C.E_CONSUME_SUCCESS)
- //ReConsumeLater it will be send back to broker
- ReConsumeLater = ConsumeStatus(C.E_RECONSUME_LATER)
-)
-
-func (status ConsumeStatus) String() string {
- switch status {
- case ConsumeSuccess:
- return "ConsumeSuccess"
- case ReConsumeLater:
- return "ReConsumeLater"
- default:
- return "Unknown"
- }
-}
-
-type defaultPushConsumer struct {
- config *PushConsumerConfig
- cconsumer *C.struct_CPushConsumer
- funcsMap sync.Map
-}
-
-func (c *defaultPushConsumer) String() string {
- topics := ""
- c.funcsMap.Range(func(key, value interface{}) bool {
- topics += key.(string) + ", "
- return true
- })
- return fmt.Sprintf("[%s, subcribed topics: [%s]]", c.config, topics)
-}
-
-func newPushConsumer(config *PushConsumerConfig) (PushConsumer, error) {
- if config == nil {
- return nil, errors.New("config is nil")
- }
- if config.GroupID == "" {
- return nil, errors.New("GroupId is empty")
- }
-
- if config.NameServer == "" && config.NameServerDomain == "" {
- return nil, errors.New("NameServer and NameServerDomain is empty")
- }
-
- consumer := &defaultPushConsumer{config: config}
- cs := C.CString(config.GroupID)
- cconsumer := C.CreatePushConsumer(cs)
- C.free(unsafe.Pointer(cs))
-
- if cconsumer == nil {
- return nil, errors.New("create PushConsumer failed")
- }
-
- var err rmqError
- if config.NameServer != "" {
- cs = C.CString(config.NameServer)
- err = rmqError(C.SetPushConsumerNameServerAddress(cconsumer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.NameServerDomain != "" {
- cs = C.CString(config.NameServerDomain)
- err = rmqError(C.SetPushConsumerNameServerDomain(cconsumer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.InstanceName != "" {
- cs = C.CString(config.InstanceName)
- err = rmqError(C.SetPushConsumerInstanceName(cconsumer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.Credentials != nil {
- ak := C.CString(config.Credentials.AccessKey)
- sk := C.CString(config.Credentials.SecretKey)
- ch := C.CString(config.Credentials.Channel)
- err = rmqError(C.SetPushConsumerSessionCredentials(cconsumer, ak, sk, ch))
- C.free(unsafe.Pointer(ak))
- C.free(unsafe.Pointer(sk))
- C.free(unsafe.Pointer(ch))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.LogC != nil {
- cs = C.CString(config.LogC.Path)
- err = rmqError(C.SetPushConsumerLogPath(cconsumer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetPushConsumerLogFileNumAndSize(cconsumer, C.int(config.LogC.FileNum), C.long(config.LogC.FileSize)))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetPushConsumerLogLevel(cconsumer, C.CLogLevel(config.LogC.Level)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.ThreadCount > 0 {
- err = rmqError(C.SetPushConsumerThreadCount(cconsumer, C.int(config.ThreadCount)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.MessageBatchMaxSize > 0 {
- err = rmqError(C.SetPushConsumerMessageBatchMaxSize(cconsumer, C.int(config.MessageBatchMaxSize)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.MaxCacheMessageSize > 0 {
- err = rmqError(C.SetPushConsumerMaxCacheMessageSize(cconsumer, C.int(config.MaxCacheMessageSize)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.MaxCacheMessageSizeInMB > 0 {
- err = rmqError(C.SetPushConsumerMaxCacheMessageSizeInMb(cconsumer, C.int(config.MaxCacheMessageSizeInMB)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.Model == BroadCasting {
- err = rmqError(C.SetPushConsumerMessageModel(cconsumer, C.BROADCASTING))
- } else if config.Model == Clustering {
- err = rmqError(C.SetPushConsumerMessageModel(cconsumer, C.CLUSTERING))
- } else {
- return nil, errors.New("model is invalid or empty")
- }
- if err != NIL {
- return nil, err
- }
-
- if config.ConsumerModel == Orderly {
- err = rmqError(C.RegisterMessageCallbackOrderly(cconsumer, (C.MessageCallBack)(unsafe.Pointer(C.callback_cgo))))
- } else if config.ConsumerModel == CoCurrently {
- err = rmqError(C.RegisterMessageCallback(cconsumer, (C.MessageCallBack)(unsafe.Pointer(C.callback_cgo))))
- } else {
- return nil, errors.New("consumer model is invalid or empty")
- }
- if err != NIL {
- return nil, err
- }
-
- consumer.cconsumer = cconsumer
- pushConsumerMap.Store(cconsumer, consumer)
- return consumer, nil
-}
-
-func (c *defaultPushConsumer) Start() error {
- err := rmqError(C.StartPushConsumer(c.cconsumer))
- if err != NIL {
- return err
- }
- return nil
-}
-
-func (c *defaultPushConsumer) Shutdown() error {
- err := rmqError(C.ShutdownPushConsumer(c.cconsumer))
-
- if err != NIL {
- return err
- }
-
- err = rmqError(C.DestroyPushConsumer(c.cconsumer))
- if err != NIL {
- return err
- }
- return nil
-}
-
-func (c *defaultPushConsumer) Subscribe(topic, expression string, consumeFunc func(msg *MessageExt) ConsumeStatus) error {
- if consumeFunc == nil {
- return errors.New("consumeFunc is nil")
- }
- err := rmqError(C.Subscribe(c.cconsumer, C.CString(topic), C.CString(expression)))
- if err != NIL {
- return err
- }
- c.funcsMap.Store(topic, consumeFunc)
- log.Infof("subscribe topic[%s] with expression[%s] successfully.", topic, expression)
- return nil
-}
diff --git a/core/push_consumer_test.go b/core/push_consumer_test.go
deleted file mode 100644
index fa5e4b5..0000000
--- a/core/push_consumer_test.go
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "errors"
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestPushConsumer_ConsumeStatus(t *testing.T) {
- assert.Equal(t, "ConsumeSuccess", ConsumeStatus(int(ConsumeSuccess)).String())
- assert.Equal(t, "ReConsumeLater", ConsumeStatus(int(ReConsumeLater)).String())
- assert.Equal(t, "Unknown", ConsumeStatus(int(-1)).String())
-}
-
-func TestPushConsumer_CreatePushConsumerFailed(t *testing.T) {
- pConfig := PushConsumerConfig{}
-
- consumer, err := newPushConsumer(nil)
- assert.Nil(t, consumer)
- assert.Equal(t, err, errors.New("config is nil"))
- consumer, err = newPushConsumer(&pConfig)
- assert.Nil(t, consumer)
- assert.Equal(t, err, errors.New("GroupId is empty"))
- pConfig.GroupID = "testGroupFailedA"
- consumer, err = newPushConsumer(&pConfig)
- assert.Nil(t, consumer)
- assert.Equal(t, err, errors.New("NameServer and NameServerDomain is empty"))
- pConfig.NameServer = "localhost:9876"
- consumer, err = newPushConsumer(&pConfig)
- assert.Nil(t, consumer)
- assert.Equal(t, err, errors.New("model is invalid or empty"))
- pConfig.Model = Clustering
- consumer, err = newPushConsumer(&pConfig)
- assert.Nil(t, consumer)
- assert.Equal(t, err, errors.New("consumer model is invalid or empty"))
- //pConfig.ConsumerModel = CoCurrently
- //pConfig.MaxCacheMessageSizeInMB = 1024
- //consumer, err = newPushConsumer(&pConfig)
- //assert.Nil(t, err)
- //assert.NotNil(t, consumer)
-}
-
-func TestPushConsumer_CreatePushConsumer(t *testing.T) {
- pConfig := PushConsumerConfig{}
- pConfig.GroupID = "testGroupSuccessA"
- pConfig.NameServer = "localhost:9876"
- pConfig.InstanceName = "testProducerA"
- pConfig.Credentials = &SessionCredentials{
- AccessKey: "AK",
- SecretKey: "SK",
- Channel: "Cloud"}
- pConfig.LogC = &LogConfig{
- Path: "/rocketmq/log",
- FileNum: 16,
- FileSize: 1 << 20,
- Level: LogLevelDebug}
- pConfig.ConsumerModel = CoCurrently
- pConfig.Model = Clustering
- pConfig.ThreadCount = 3
- pConfig.MessageBatchMaxSize = 1
- pConfig.MaxCacheMessageSize = 1000
- //pConfig.MaxCacheMessageSizeInMB = 1024
- consumer, err := newPushConsumer(&pConfig)
- assert.Nil(t, err)
- assert.NotNil(t, consumer)
-}
-func callbackTest(msg *MessageExt) ConsumeStatus {
- return ReConsumeLater
-}
-func TestPushConsumer_CreatePushConsumerSubscribe(t *testing.T) {
- pConfig := PushConsumerConfig{}
- pConfig.GroupID = "testGroup"
- pConfig.NameServer = "localhost:9876"
- pConfig.InstanceName = "testProducer"
- pConfig.Credentials = &SessionCredentials{
- AccessKey: "AK",
- SecretKey: "SK",
- Channel: "Cloud"}
- pConfig.LogC = &LogConfig{
- Path: "/rocketmq/log",
- FileNum: 16,
- FileSize: 1 << 20,
- Level: LogLevelDebug}
- pConfig.ConsumerModel = CoCurrently
- pConfig.Model = Clustering
- pConfig.ThreadCount = 3
- pConfig.MessageBatchMaxSize = 1
- pConfig.MaxCacheMessageSize = 1000
- //pConfig.MaxCacheMessageSizeInMB = 1024
- consumer, err := newPushConsumer(&pConfig)
- assert.Nil(t, err)
- assert.NotNil(t, consumer)
- err = consumer.Subscribe("Topic", "exp", nil)
- assert.Equal(t, err, errors.New("consumeFunc is nil"))
- err = consumer.Subscribe("Topic", "exp", callbackTest)
- assert.Nil(t, err)
-}
diff --git a/core/queue.go b/core/queue.go
deleted file mode 100644
index 47f10ef..0000000
--- a/core/queue.go
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-import "fmt"
-
-// MessageQueue the queue of the message
-type MessageQueue struct {
- Topic string
- Broker string
- ID int
-}
-
-func (q *MessageQueue) String() string {
- return fmt.Sprintf("broker:%s, topic:%s, id:%d", q.Broker, q.Topic, q.ID)
-}
diff --git a/core/queue_selector.go b/core/queue_selector.go
deleted file mode 100644
index cb660ac..0000000
--- a/core/queue_selector.go
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-import "C"
-import (
- "strconv"
- "sync"
- "unsafe"
-)
-
-var selectors = selectorHolder{selectors: map[int]*messageQueueSelectorWrapper{}}
-
-//export queueSelectorCallback
-func queueSelectorCallback(size int, selectorKey unsafe.Pointer) int {
- s, ok := selectors.getAndDelete(*(*int)(selectorKey))
- if !ok {
- panic("BUG: not register the selector with key:" + strconv.Itoa(*(*int)(selectorKey)))
- }
- return s.Select(size)
-}
-
-type messageQueueSelectorWrapper struct {
- selector MessageQueueSelector
-
- m *Message
- arg interface{}
-}
-
-func (w *messageQueueSelectorWrapper) Select(size int) int {
- return w.selector.Select(size, w.m, w.arg)
-}
-
-// MessageQueueSelector select one message queue
-type MessageQueueSelector interface {
- Select(size int, m *Message, arg interface{}) int
-}
-
-type selectorHolder struct {
- sync.Mutex
-
- selectors map[int]*messageQueueSelectorWrapper
- key int
-}
-
-func (s *selectorHolder) put(selector *messageQueueSelectorWrapper) (key int) {
- s.Lock()
- key = s.key
- s.selectors[key] = selector
- s.key++
- s.Unlock()
- return
-}
-
-func (s *selectorHolder) getAndDelete(key int) (*messageQueueSelectorWrapper, bool) {
- s.Lock()
- selector, ok := s.selectors[key]
- if ok {
- delete(s.selectors, key)
- }
- s.Unlock()
-
- return selector, ok
-}
diff --git a/core/queue_selector_test.go b/core/queue_selector_test.go
deleted file mode 100644
index 74fff80..0000000
--- a/core/queue_selector_test.go
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-type mockMessageQueueSelector struct {
- arg interface{}
- m *Message
- size int
-
- selectRet int
-}
-
-func (m *mockMessageQueueSelector) Select(size int, msg *Message, arg interface{}) int {
- m.arg, m.m, m.size = arg, msg, size
- return m.selectRet
-}
-
-func TestWrapper(t *testing.T) {
- s := &mockMessageQueueSelector{selectRet: 2}
- w := &messageQueueSelectorWrapper{selector: s, m: &Message{}, arg: 3}
-
- assert.Equal(t, 2, w.Select(4))
- assert.Equal(t, w.m, s.m)
- v, ok := s.arg.(int)
- assert.True(t, ok)
- assert.Equal(t, 3, v)
-}
-
-func TestSelectorHolder(t *testing.T) {
- s := &messageQueueSelectorWrapper{}
-
- key := selectors.put(s)
- assert.Equal(t, 0, key)
-
- key = selectors.put(s)
- assert.Equal(t, 1, key)
-
- assert.Equal(t, 2, len(selectors.selectors))
-
- ss, ok := selectors.getAndDelete(0)
- assert.Equal(t, s, ss)
- assert.True(t, ok)
-
- ss, ok = selectors.getAndDelete(1)
- assert.Equal(t, s, ss)
- assert.True(t, ok)
-
- assert.Equal(t, 0, len(selectors.selectors))
-}
diff --git a/core/queue_test.go b/core/queue_test.go
deleted file mode 100644
index f172386..0000000
--- a/core/queue_test.go
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestMessageQueue_String(t *testing.T) {
- mq := MessageQueue{
- Topic: "testTopic",
- Broker: "BrokerA",
- ID: 1}
- expect := "broker:BrokerA, topic:testTopic, id:1"
- assert.Equal(t, expect, mq.String())
-}
diff --git a/core/transaction_callback.go b/core/transaction_callback.go
deleted file mode 100644
index ddab6aa..0000000
--- a/core/transaction_callback.go
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#cgo LDFLAGS: -L/usr/local/lib -lrocketmq
-
-#include <rocketmq/CMessage.h>
-#include <rocketmq/CMessageExt.h>
-#include <rocketmq/CProducer.h>
-*/
-import "C"
-import "sync"
-
-var transactionProducerMap sync.Map
-
-//export localTransactionExecutorCallback
-func localTransactionExecutorCallback(cproducer *C.CProducer, msg *C.CMessage, arg interface{}) C.int {
- producer, exist := transactionProducerMap.Load(cproducer)
- if !exist {
- return C.int(UnknownTransaction)
- }
-
- message := cMsgToGo(msg)
- listenerWrap, exist := producer.(*defaultTransactionProducer).listenerFuncsMap.Load(cproducer)
- if !exist {
- return C.int(UnknownTransaction)
- }
- status := listenerWrap.(TransactionLocalListener).Execute(message, arg)
- return C.int(status)
-}
-
-//export localTransactionCheckerCallback
-func localTransactionCheckerCallback(cproducer *C.CProducer, msg *C.CMessageExt, arg interface{}) C.int {
- producer, exist := transactionProducerMap.Load(cproducer)
- if !exist {
- return C.int(UnknownTransaction)
- }
-
- message := cmsgExtToGo(msg)
- listener, exist := producer.(*defaultTransactionProducer).listenerFuncsMap.Load(cproducer)
- if !exist {
- return C.int(UnknownTransaction)
- }
- status := listener.(TransactionLocalListener).Check(message, arg)
- return C.int(status)
-}
diff --git a/core/transaction_producer.go b/core/transaction_producer.go
deleted file mode 100644
index e79f2c2..0000000
--- a/core/transaction_producer.go
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-/*
-#cgo LDFLAGS: -L/usr/local/lib/ -lrocketmq
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <rocketmq/CMessage.h>
-#include <rocketmq/CProducer.h>
-#include <rocketmq/CSendResult.h>
-#include <rocketmq/CTransactionStatus.h>
-
-extern int localTransactionCheckerCallback(CProducer *producer, CMessageExt *msg,void *userData);
-int transactionChecker_cgo(CProducer *producer, CMessageExt *msg, void *userData) {
- return localTransactionCheckerCallback(producer, msg, userData);
-}
-
-extern int localTransactionExecutorCallback(CProducer *producer, CMessage *msg,void *userData);
-int transactionExecutor_cgo(CProducer *producer, CMessage *msg, void *userData) {
- return localTransactionExecutorCallback(producer, msg, userData);
-}
-*/
-import "C"
-import (
- "errors"
- log "github.com/sirupsen/logrus"
- "sync"
- "unsafe"
-)
-
-//TransactionStatus check the status if commit or rollback
-type TransactionStatus int
-
-//TransactionStatus check the status if commit or rollback
-const (
- CommitTransaction = TransactionStatus(C.E_COMMIT_TRANSACTION)
- RollbackTransaction = TransactionStatus(C.E_ROLLBACK_TRANSACTION)
- UnknownTransaction = TransactionStatus(C.E_UNKNOWN_TRANSACTION)
-)
-
-func (status TransactionStatus) String() string {
- switch status {
- case CommitTransaction:
- return "CommitTransaction"
- case RollbackTransaction:
- return "RollbackTransaction"
- case UnknownTransaction:
- return "UnknownTransaction"
- default:
- return "UnknownTransaction"
- }
-}
-func newDefaultTransactionProducer(config *ProducerConfig, listener TransactionLocalListener, arg interface{}) (*defaultTransactionProducer, error) {
- if config == nil {
- return nil, errors.New("config is nil")
- }
-
- if config.GroupID == "" {
- return nil, errors.New("GroupId is empty")
- }
-
- if config.NameServer == "" && config.NameServerDomain == "" {
- return nil, errors.New("NameServer and NameServerDomain is empty")
- }
-
- producer := &defaultTransactionProducer{config: config}
- cs := C.CString(config.GroupID)
- var cproducer *C.struct_CProducer
-
- cproducer = C.CreateTransactionProducer(cs, (C.CLocalTransactionCheckerCallback)(unsafe.Pointer(C.transactionChecker_cgo)), unsafe.Pointer(&arg))
-
- C.free(unsafe.Pointer(cs))
-
- if cproducer == nil {
- return nil, errors.New("create transaction Producer failed")
- }
-
- var err rmqError
- if config.NameServer != "" {
- cs = C.CString(config.NameServer)
- err = rmqError(C.SetProducerNameServerAddress(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.NameServerDomain != "" {
- cs = C.CString(config.NameServerDomain)
- err = rmqError(C.SetProducerNameServerDomain(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.InstanceName != "" {
- cs = C.CString(config.InstanceName)
- err = rmqError(C.SetProducerInstanceName(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.Credentials != nil {
- ak := C.CString(config.Credentials.AccessKey)
- sk := C.CString(config.Credentials.SecretKey)
- ch := C.CString(config.Credentials.Channel)
- err = rmqError(C.SetProducerSessionCredentials(cproducer, ak, sk, ch))
-
- C.free(unsafe.Pointer(ak))
- C.free(unsafe.Pointer(sk))
- C.free(unsafe.Pointer(ch))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.LogC != nil {
- cs = C.CString(config.LogC.Path)
- err = rmqError(C.SetProducerLogPath(cproducer, cs))
- C.free(unsafe.Pointer(cs))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetProducerLogFileNumAndSize(cproducer, C.int(config.LogC.FileNum), C.long(config.LogC.FileSize)))
- if err != NIL {
- return nil, err
- }
-
- err = rmqError(C.SetProducerLogLevel(cproducer, C.CLogLevel(config.LogC.Level)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.SendMsgTimeout > 0 {
- err = rmqError(C.SetProducerSendMsgTimeout(cproducer, C.int(config.SendMsgTimeout)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.CompressLevel > 0 {
- err = rmqError(C.SetProducerCompressLevel(cproducer, C.int(config.CompressLevel)))
- if err != NIL {
- return nil, err
- }
- }
-
- if config.MaxMessageSize > 0 {
- err = rmqError(C.SetProducerMaxMessageSize(cproducer, C.int(config.MaxMessageSize)))
- if err != NIL {
- return nil, err
- }
- }
-
- producer.cproducer = cproducer
- transactionProducerMap.Store(cproducer, producer)
- producer.listenerFuncsMap.Store(cproducer, listener)
- return producer, nil
-}
-
-type defaultTransactionProducer struct {
- config *ProducerConfig
- cproducer *C.struct_CProducer
- listenerFuncsMap sync.Map
-}
-
-func (p *defaultTransactionProducer) String() string {
- return p.config.String()
-}
-
-// Start the producer.
-func (p *defaultTransactionProducer) Start() error {
- err := rmqError(C.StartProducer(p.cproducer))
- if err != NIL {
- return err
- }
- return nil
-}
-
-// Shutdown the producer.
-func (p *defaultTransactionProducer) Shutdown() error {
- err := rmqError(C.ShutdownProducer(p.cproducer))
-
- if err != NIL {
- return err
- }
-
- err = rmqError(int(C.DestroyProducer(p.cproducer)))
- if err != NIL {
- return err
- }
-
- return err
-}
-
-func (p *defaultTransactionProducer) SendMessageTransaction(msg *Message, arg interface{}) (*SendResult, error) {
- cmsg := goMsgToC(msg)
- defer C.DestroyMessage(cmsg)
-
- var sr C.struct__SendResult_
- err := rmqError(C.SendMessageTransaction(p.cproducer, cmsg, (C.CLocalTransactionExecutorCallback)(unsafe.Pointer(C.transactionExecutor_cgo)), unsafe.Pointer(&arg), &sr))
- if err != NIL {
- log.Warnf("send message error, error is: %s", err.Error())
- return nil, err
- }
-
- result := &SendResult{}
- result.Status = SendStatus(sr.sendStatus)
- result.MsgId = C.GoString(&sr.msgId[0])
- result.Offset = int64(sr.offset)
- return result, nil
-}
diff --git a/core/transaction_producer_test.go b/core/transaction_producer_test.go
deleted file mode 100644
index 3ef9e46..0000000
--- a/core/transaction_producer_test.go
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "errors"
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestTransactionProducer_TransactionStatus(t *testing.T) {
- assert.Equal(t, "CommitTransaction", TransactionStatus(int(CommitTransaction)).String())
- assert.Equal(t, "RollbackTransaction", TransactionStatus(int(RollbackTransaction)).String())
- assert.Equal(t, "UnknownTransaction", TransactionStatus(int(UnknownTransaction)).String())
- assert.Equal(t, "UnknownTransaction", TransactionStatus(int(-1)).String())
-}
-
-func TestTransactionProducer_CreateProducerFailed(t *testing.T) {
- pConfig := ProducerConfig{}
-
- producer, err := newDefaultTransactionProducer(nil, nil, nil)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("config is nil"))
- producer, err = newDefaultTransactionProducer(&pConfig, nil, nil)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("GroupId is empty"))
- pConfig.GroupID = "testGroup"
- producer, err = newDefaultTransactionProducer(&pConfig, nil, nil)
- assert.Nil(t, producer)
- assert.Equal(t, err, errors.New("NameServer and NameServerDomain is empty"))
-}
-
-type MyTransactionLocalListener struct {
-}
-
-func (l *MyTransactionLocalListener) Execute(m *Message, arg interface{}) TransactionStatus {
- return UnknownTransaction
-}
-func (l *MyTransactionLocalListener) Check(m *MessageExt, arg interface{}) TransactionStatus {
- return CommitTransaction
-}
-func TestTransactionProducer_CreateProducer(t *testing.T) {
- pConfig := ProducerConfig{}
- pConfig.GroupID = "testGroup"
- pConfig.NameServer = "localhost:9876"
- pConfig.InstanceName = "testProducer"
- pConfig.Credentials = &SessionCredentials{
- AccessKey: "AK",
- SecretKey: "SK",
- Channel: "Cloud"}
- pConfig.LogC = &LogConfig{
- Path: "/rocketmq/log",
- FileNum: 16,
- FileSize: 1 << 20,
- Level: LogLevelDebug}
- pConfig.SendMsgTimeout = 30
- pConfig.CompressLevel = 4
- pConfig.MaxMessageSize = 1024
- listener := &MyTransactionLocalListener{}
- producer, err := newDefaultTransactionProducer(&pConfig, listener, nil)
- assert.Nil(t, err)
- assert.NotEmpty(t, producer)
-}
diff --git a/core/utils.go b/core/utils.go
deleted file mode 100644
index c2e94c3..0000000
--- a/core/utils.go
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-import "fmt"
-
-func strJoin(str, key string, value interface{}) string {
- if key == "" || value == "" {
- return str
- }
-
- return str + key + ": " + fmt.Sprint(value) + ", "
-}
diff --git a/core/utils_test.go b/core/utils_test.go
deleted file mode 100644
index 93b0e0c..0000000
--- a/core/utils_test.go
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestUtils_strJoin(test *testing.T) {
- assert.Equal(test, "TestKeyA: ValueA, ", strJoin("Test", "KeyA", "ValueA"))
-}
diff --git a/core/version.go b/core/version.go
deleted file mode 100644
index fd384db..0000000
--- a/core/version.go
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package rocketmq
-
-//GoClientVersion const strings for version
-const GoClientVersion = "Go Client V1.2.4, Support CPP Core:V1.2.X"
-
-//GetVersion return go version strings
-func GetVersion() (version string) {
- return GoClientVersion
-}
diff --git a/core/version_test.go b/core/version_test.go
deleted file mode 100644
index a99a22d..0000000
--- a/core/version_test.go
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package rocketmq
-
-import (
- "github.com/stretchr/testify/assert"
- "testing"
-)
-
-func TestGetVersion(test *testing.T) {
- expect := "Go Client V1.2.4, Support CPP Core:V1.2.X"
- assert.Equal(test, expect, GetVersion())
-}
diff --git a/doc/Introduction.md b/doc/Introduction.md
deleted file mode 100644
index a4f6339..0000000
--- a/doc/Introduction.md
+++ /dev/null
@@ -1,173 +0,0 @@
-## Prerequisites
-
-### Install `librocketmq`
-rocketmq-client-go is a lightweight wrapper around [rocketmq-client-cpp](https://github.com/apache/rocketmq-client-cpp), so you need to install
-the `librocketmq` first. you can install it using binary release or build it from source code manually.
-
-#### Download by binary release.
-download specific [cpp release version](https://github.com/apache/rocketmq-client-cpp/releases) according you OS, here we take [rocketmq-client-cpp-2.0.1](https://github.com/apache/rocketmq-client-cpp/releases/tag/2.0.1) as an example.
-- centos
-
- take centos7 as an example, you can install the library in centos6 by the same method.
- ```bash
- wget https://github.com/apache/rocketmq-client-cpp/releases/download/2.0.1/rocketmq-client-cpp-2.0.1-centos7.x86_64.rpm
- sudo rpm -ivh rocketmq-client-cpp-2.0.1-centos7.x86_64.rpm
- ```
-- debian
- ```bash
- wget https://github.com/apache/rocketmq-client-cpp/releases/download/2.0.1/rocketmq-client-cpp-2.0.1.amd64.deb
- sudo dpkg -i rocketmq-client-cpp-2.0.1.amd64.deb
- ```
-- macOS
- ```bash
- wget https://github.com/apache/rocketmq-client-cpp/releases/download/2.0.1/rocketmq-client-cpp-2.0.1-bin-release.darwin.tar.gz
- tar -xzf rocketmq-client-cpp-2.0.1-bin-release.darwin.tar.gz
- cd rocketmq-client-cpp
- mkdir /usr/local/include/rocketmq
- cp include/* /usr/local/include/rocketmq
- cp lib/* /usr/local/lib
- ```
-#### Build from source
-you can also build it manually from source according to [Build and Install](https://github.com/apache/rocketmq-client-cpp/tree/master#build-and-install)
-### Gcc install
-gcc/g++ 4.8+ is needed in the cgo compile, please make sure is it installed in you machine.
-### SDK install
-1. Go Version: 1.10 or later
-2. `go get github.com/apache/rocketmq-client-go`
-
-## How to use
-
-- import package
- ```
- import rocketmq "github.com/apache/rocketmq-client-go/core"
- ```
-- Send message
- ```go
- func SendMessagge(){
- producer := rocketmq.NewProducer(config)
- producer.Start()
- defer producer.Shutdown()
- fmt.Printf("Producer: %s started... \n", producer)
- for i := 0; i < 100; i++ {
- msg := fmt.Sprintf("%s-*d", *body, i)
- result, err := producer.SendMessageSync(&rocketmq.Message{Topic: "test", Body: msg})
- if err != nil {
- fmt.Println("Error:", err)
- }
- fmt.Printf("send message: %s result: %s\n", msg, result)
- }
- }
- ```
-- Send ordered message
- ```go
- func sendMessageOrderlyByShardingKey(config *rocketmq.ProducerConfig) {
- producer, err := rocketmq.NewProducer(config)
- if err != nil {
- fmt.Println("create Producer failed, error:", err)
- return
- }
-
- producer.Start()
- defer producer.Shutdown()
- for i := 0; i < 1000; i++ {
- msg := fmt.Sprintf("%s-%d", "Hello Lite Orderly Message", i)
- r, err := producer.SendMessageOrderlyByShardingKey(
- &rocketmq.Message{Topic: "YourOrderLyTopicXXXXXXXX", Body: msg}, "ShardingKey" /*orderID*/)
- if err != nil {
- println("Send Orderly Message Error:", err)
- }
- fmt.Printf("send orderly message result:%+v\n", r)
- time.Sleep(time.Duration(1) * time.Second)
- }
-
- }
- ```
-- Push Consumer
- ```go
- func ConsumeWithPush(config *rocketmq.PushConsumerConfig) {
-
- consumer, err := rocketmq.NewPushConsumer(config)
- if err != nil {
- println("create Consumer failed, error:", err)
- return
- }
-
- ch := make(chan interface{})
- var count = (int64)(*amount)
- // MUST subscribe topic before consumer started.
- consumer.Subscribe("test", "*", func(msg *rocketmq.MessageExt) rocketmq.ConsumeStatus {
- fmt.Printf("A message received: \"%s\" \n", msg.Body)
- if atomic.AddInt64(&count, -1) <= 0 {
- ch <- "quit"
- }
- return rocketmq.ConsumeSuccess
- })
-
- err = consumer.Start()
- if err != nil {
- println("consumer start failed,", err)
- return
- }
-
- fmt.Printf("consumer: %s started...\n", consumer)
- <-ch
- err = consumer.Shutdown()
- if err != nil {
- println("consumer shutdown failed")
- return
- }
- println("consumer has shutdown.")
- }
- ```
-- Pull Consumer
- ```go
- func ConsumeWithPull(config *rocketmq.PullConsumerConfig, topic string) {
-
- consumer, err := rocketmq.NewPullConsumer(config)
- if err != nil {
- fmt.Printf("new pull consumer error:%s\n", err)
- return
- }
-
- err = consumer.Start()
- if err != nil {
- fmt.Printf("start consumer error:%s\n", err)
- return
- }
- defer consumer.Shutdown()
-
- mqs := consumer.FetchSubscriptionMessageQueues(topic)
- fmt.Printf("fetch subscription mqs:%+v\n", mqs)
-
- total, offsets, now := 0, map[int]int64{}, time.Now()
-
- PULL:
- for {
- for _, mq := range mqs {
- pr := consumer.Pull(mq, "*", offsets[mq.ID], 32)
- total += len(pr.Messages)
- fmt.Printf("pull %s, result:%+v\n", mq.String(), pr)
-
- switch pr.Status {
- case rocketmq.PullNoNewMsg:
- break PULL
- case rocketmq.PullFound:
- fallthrough
- case rocketmq.PullNoMatchedMsg:
- fallthrough
- case rocketmq.PullOffsetIllegal:
- offsets[mq.ID] = pr.NextBeginOffset
- case rocketmq.PullBrokerTimeout:
- fmt.Println("broker timeout occur")
- }
- }
- }
-
- var timePerMessage time.Duration
- if total > 0 {
- timePerMessage = time.Since(now) / time.Duration(total)
- }
- fmt.Printf("total message:%d, per message time:%d\n", total, timePerMessage)
- }
- ```
-- [Full example](../examples)
diff --git a/docs/Introduction.md b/docs/Introduction.md
new file mode 100644
index 0000000..2e32356
--- /dev/null
+++ b/docs/Introduction.md
@@ -0,0 +1,95 @@
+## How to use
+
+### go mod
+```
+require (
+ github.com/apache/rocketmq-client-go/v2 v2.0.0-rc1
+)
+```
+
+### Set Logger
+Go Client define the `Logger` interface for log output, user can specify implementation of private.
+in default, client use `logrus`.
+```go
+rlog.SetLogger(Logger)
+```
+
+### Send message
+#### Interface
+```go
+Producer interface {
+ Start() error
+ Shutdown() error
+ SendSync(context.Context, *primitive.Message) (*internal.SendResult, error)
+ SendOneWay(context.Context, *primitive.Message) error
+}
+```
+
+#### Examples
+- create a new `Producer` instance
+```go
+opt := producer.ProducerOptions{
+ NameServerAddr: "127.0.0.1:9876",
+ RetryTimesWhenSendFailed: 2,
+}
+p := producer.NewProducer(opt)
+```
+
+- start the producer
+```go
+err := p.Start()
+```
+
+- send message with sync
+```go
+result, err := p.SendSync(context.Background(), &primitive.Message{
+ Topic: "test",
+ Body: []byte("Hello RocketMQ Go Client!"),
+})
+
+// do something with result
+```
+
+- or send message with oneway
+```go
+err := p.SendOneWay(context.Background(), &primitive.Message{
+ Topic: "test",
+ Body: []byte("Hello RocketMQ Go Client!"),
+})
+```
+Full examples: [producer](../examples/producer)
+
+### Consume Message
+alpha1 only support `PushConsumer`
+
+#### Interface
+```go
+PushConsumer interface {
+ Start() error
+ Shutdown()
+ Subscribe(topic string, selector MessageSelector,
+ f func(*ConsumeMessageContext, []*primitive.MessageExt) (ConsumeResult, error)) error
+}
+```
+
+#### Usage
+- Create a `PushConsumer` instance
+```go
+c := consumer.NewPushConsumer("testGroup", consumer.ConsumerOption{
+ NameServerAddr: "127.0.0.1:9876",
+ ConsumerModel: consumer.Clustering,
+ FromWhere: consumer.ConsumeFromFirstOffset,
+})
+```
+
+- Subscribe a topic(only support one topic now), and define your consuming function
+```go
+err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx *consumer.ConsumeMessageContext,
+ msgs []*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Println(msgs)
+ return consumer.ConsumeSuccess, nil
+})
+```
+- start the consumer(**NOTE: MUST after subscribe**)
+
+Full examples: [consumer](../examples/consumer)
diff --git a/docs/client-design.gliffy b/docs/client-design.gliffy
new file mode 100644
index 0000000..418f4d8
--- /dev/null
+++ b/docs/client-design.gliffy
@@ -0,0 +1 @@
+{"contentType":"application/gliffy+json","version":"1.1","metadata":{"title":"untitled","revision":0,"exportBorder":false},"embeddedResources":{"index":0,"resources":[]},"stage":{"objects":[{"x":31,"y":44,"rotation":0,"id":57,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":27,"lockAspectRatio":false,"lockShape":false,"order":57,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 24px; font-family: Arial; color: rgb(106, 168, 79); white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 27px;\">Architecture</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":0,"y":10,"rotation":0,"id":51,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":710,"height":360,"lockAspectRatio":false,"lockShape":false,"order":0,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":1,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[],"linkMap":[]},{"x":420,"y":187,"rotation":0,"id":44,"uid":"com.gliffy.shape.basic.basic_v1.default.line","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":25,"graphic":{"type":"Line","Line":{"strokeWidth":2,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":1,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":false,"interpolationType":"linear","cornerRadius":null,"controlPath":[[-3,-2],[37,-2]],"lockSegments":{}}},"children":[],"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":11,"px":1,"py":0.5}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":13,"px":0,"py":0.5}}},"linkMap":[]},{"x":104,"y":171,"rotation":0,"id":42,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":28,"lockAspectRatio":false,"lockShape":false,"order":24,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">1. API wrapper\n</span></p><p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">2. connections manager</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":451,"y":61,"rotation":0,"id":37,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":23,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">3. offset storage</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":433,"y":46,"rotation":0,"id":36,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":22,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">2. balance</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":471,"y":30,"rotation":0,"id":35,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":180,"height":14,"lockAspectRatio":false,"lockShape":false,"order":21,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">1. push/pull(push based pull)</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":252,"y":61,"rotation":0,"id":28,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":20,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">2. muti-producers</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":251,"y":42,"rotation":0,"id":27,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":19,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">1. queue selector</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":154,"y":272,"rotation":0,"id":21,"uid":"com.gliffy.shape.uml.uml_v1.default.message","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":18,"graphic":{"type":"Line","Line":{"strokeWidth":1,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":2,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":true,"interpolationType":"linear","cornerRadius":null,"controlPath":[[278,-52],[278,-45.33333333333334],[278,-38.66666666666666],[278,-32]],"lockSegments":{}}},"children":null,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":7,"px":0.5,"py":1}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":0,"px":0.5,"py":0}}},"linkMap":[]},{"x":154,"y":272,"rotation":0,"id":20,"uid":"com.gliffy.shape.uml.uml_v1.default.message","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":17,"graphic":{"type":"Line","Line":{"strokeWidth":1,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":2,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":true,"interpolationType":"linear","cornerRadius":null,"controlPath":[[388,-152],[388,-137],[352.5584412271571,-137],[352.5584412271571,-122]],"lockSegments":{}}},"children":null,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":17,"px":0.5,"py":1}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":7,"px":0.7071067811865476,"py":0}}},"linkMap":[]},{"x":144,"y":262,"rotation":0,"id":19,"uid":"com.gliffy.shape.uml.uml_v1.default.message","width":100,"height":100,"lockAspectRatio":false,"lockShape":false,"order":16,"graphic":{"type":"Line","Line":{"strokeWidth":1,"strokeColor":"#000000","fillColor":"none","dashStyle":null,"startArrow":0,"endArrow":2,"startArrowRotation":"auto","endArrowRotation":"auto","ortho":true,"interpolationType":"linear","cornerRadius":null,"controlPath":[[183,-142],[183,-127],[213.44155877284288,-127],[213.44155877284288,-112]],"lockSegments":{}}},"children":null,"constraints":{"constraints":[],"startConstraint":{"type":"StartPositionConstraint","StartPositionConstraint":{"nodeId":15,"px":0.5,"py":1}},"endConstraint":{"type":"EndPositionConstraint","EndPositionConstraint":{"nodeId":7,"px":0.2928932188134524,"py":0}}},"linkMap":[]},{"x":472,"y":80,"rotation":0,"id":17,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":140,"height":40,"lockAspectRatio":false,"lockShape":false,"order":14,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":1.9999999999999998,"y":0,"rotation":0,"id":18,"uid":null,"width":135.99999999999997,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">consumer</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":252,"y":80,"rotation":0,"id":15,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":150,"height":40,"lockAspectRatio":false,"lockShape":false,"order":12,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.142857142857143,"y":0,"rotation":0,"id":16,"uid":null,"width":145.71428571428572,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">producer</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":457,"y":165,"rotation":0,"id":13,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":140,"height":40,"lockAspectRatio":false,"lockShape":false,"order":10,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":1.9999999999999998,"y":0,"rotation":0,"id":14,"uid":null,"width":135.99999999999997,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">route</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":267,"y":165,"rotation":0,"id":11,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":150,"height":40,"lockAspectRatio":false,"lockShape":false,"order":8,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.142857142857143,"y":0,"rotation":0,"id":12,"uid":null,"width":145.71428571428572,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">manager</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":237,"y":248,"rotation":0,"id":9,"uid":"com.gliffy.shape.basic.basic_v1.default.text","width":150,"height":14,"lockAspectRatio":false,"lockShape":false,"order":7,"graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">remote</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null,"linkMap":[]},{"x":252.00000000000003,"y":150,"rotation":0,"id":7,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":359.99999999999994,"height":70,"lockAspectRatio":false,"lockShape":false,"order":6,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":true,"state":0,"shadowX":4,"shadowY":4,"opacity":1}},"children":[],"linkMap":[]},{"x":342,"y":299,"rotation":0,"id":5,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":180,"height":40,"lockAspectRatio":false,"lockShape":false,"order":4,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.5714285714285716,"y":0,"rotation":0,"id":6,"uid":null,"width":174.8571428571429,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">codec</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":342,"y":248,"rotation":0,"id":2,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":180,"height":40,"lockAspectRatio":false,"lockShape":false,"order":2,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":false,"state":0,"shadowX":0,"shadowY":0,"opacity":1}},"children":[{"x":2.5714285714285716,"y":0,"rotation":0,"id":4,"uid":null,"width":174.8571428571429,"height":14,"lockAspectRatio":false,"lockShape":false,"order":"auto","graphic":{"type":"Text","Text":{"tid":null,"valign":"middle","overflow":"none","vposition":"none","hposition":"none","html":"<p style=\"text-align:center;\"><span style=\"font-size: 12px; font-family: Arial; white-space: pre-wrap; font-weight: bold; text-decoration: none; line-height: 14px; color: rgb(0, 0, 0);\">client</span></p>","paddingLeft":2,"paddingRight":2,"paddingBottom":2,"paddingTop":2}},"children":null}],"linkMap":[]},{"x":287,"y":240,"rotation":0,"id":0,"uid":"com.gliffy.shape.uml.uml_v1.default.simple_class","width":290,"height":110,"lockAspectRatio":false,"lockShape":false,"order":1,"graphic":{"type":"Shape","Shape":{"tid":"com.gliffy.stencil.rectangle.basic_v1","strokeWidth":2,"strokeColor":"#000000","fillColor":"#FFFFFF","gradient":false,"dropShadow":true,"state":0,"shadowX":4,"shadowY":4,"opacity":1}},"children":[],"linkMap":[]}],"background":"#FFFFFF","width":710,"height":370,"maxWidth":5000,"maxHeight":5000,"nodeIndex":59,"autoFit":true,"exportBorder":false,"gridOn":true,"snapToGrid":true,"drawingGuidesOn":true,"pageBreaksOn":false,"printGridOn":false,"printPaper":"LETTER","printShrinkToFit":false,"printPortrait":true,"shapeStyles":{},"lineStyles":{"global":{"endArrow":1}},"textStyles":{},"themeData":null}}
\ No newline at end of file
diff --git a/docs/feature.md b/docs/feature.md
new file mode 100644
index 0000000..795ffac
--- /dev/null
+++ b/docs/feature.md
@@ -0,0 +1,73 @@
+# Feature
+
+## Producer
+
+### MessageType
+- [x] NormalMessage
+- [ ] TransactionMessage
+- [ ] DelayMessage
+
+### SendWith
+- [x] Sync
+- [ ] Async
+- [x] OneWay
+
+### Other
+- [ ] Config
+- [ ] MessageId Generate
+- [ ] CompressMsg
+- [ ] LoadBalance
+- [ ] DefaultTopic
+- [ ] VipChannel
+- [ ] Retry
+- [ ] Hook
+- [ ] CheckRequestQueue
+- [ ] MQFaultStrategy
+
+## Consumer
+
+### ReceiveType
+- [x] Push
+- [ ] Pull
+
+### ConsumingType
+- [x] Concurrently
+- [ ] Orderly
+
+### MessageModel
+- [x] CLUSTERING
+- [x] BROADCASTING
+
+### AllocateMessageQueueStrategy
+- [x] AllocateMessageQueueAveragely
+- [x] AllocateMessageQueueAveragelyByCircle
+- [X] AllocateMessageQueueByConfig
+- [X] AllocateMessageQueueByMachineRoom
+
+### Other
+- [x] Rebalance
+- [x] Flow Control
+- [ ] compress
+- [x] ConsumeFromWhere
+- [ ] Retry(sendMessageBack)
+- [ ] Hook
+
+## Common
+- [ ] PollNameServer
+- [x] Heartbeat
+- [x] UpdateTopicRouteInfoFromNameServer
+- [ ] CleanOfflineBroker
+- [ ] ClearExpiredMessage(form consumer consumeMessageService)
+
+## Remoting
+- [x] API
+ - [x] InvokeSync
+ - [x] InvokeAsync
+ - [x] InvokeOneWay
+- [x] Serialize
+ - [x] JSON
+ - [x] ROCKETMQ
+- [ ] Other
+ - [ ] VIPChannel
+ - [ ] RPCHook
+
\ No newline at end of file
diff --git a/docs/images/client-design.png b/docs/images/client-design.png
new file mode 100644
index 0000000..97b86ef
--- /dev/null
+++ b/docs/images/client-design.png
Binary files differ
diff --git a/docs/zh/native-design_zh.md b/docs/zh/native-design_zh.md
new file mode 100644
index 0000000..dd407a4
--- /dev/null
+++ b/docs/zh/native-design_zh.md
@@ -0,0 +1,114 @@
+# RocketMQ Go Client Design Draft
+
+## Architecture
+
+### Overview
+
+
+### Description
+在RocketMQ Java Client的实现里面，代码耦合了大量的admin方面的功能， 其为了尽可能的提高代码复用率，代码的依赖关系较为复杂、接口的设计比
+较重、语义的界限不够清晰。因此，为了避免简单的对Java代码进行翻译，故Go客户端进行了重新的设计，剥离了admin相关的逻辑。整体如上面的图所示，在逻辑层次上，按照请求
+顺序，从下到上总共分为三层：
+- remote层：client网络通信和私有协议层，将到每个到服务端(NameServer或Broker)的连接实体抽象为一个client结构，并在这里实现了网络数据的
+序列/反序列化。
+- 公共层：由于remote层对上层只暴露了`Sync/Async/Oneway`三个接口，所以对于特定的Request/Response、连接的管理、路由等信息的处理和维护、
+其它`producer/consumer`共用的逻辑，均在这里实现。
+- 业务逻辑层：在这里实现各种producer、consumer的语义。所有producer、consumer专有的逻辑，均放到这里实现，如`queue selector`,
+`consume balance`, `offset storage`等。除了基础的数据结构外，producer和consumer之间内部的代码不能进行复用。
+
+
+## 设计目标
+0. 兼容cgo版本已经暴露出去的API
+1. 实现语义清晰、轻量的API接口
+2. 依赖关系清晰简单，设计和实现要正交
+
+## 目录结构
+### 源码
+- producer：producer相关逻辑
+- consumer：consumer相关逻辑
+- common（可改名）：连接管理和路由管理相关的通用逻辑
+- remote：网络通信和序列化
+
+### 其它
+- benchmark：压力测试相关代码
+- core：1.2版本cgo的代码库，该目录下的代码将会被移除，只保留API进行兼容，并会被标记为`Deprecated`
+- docs：文档，包括面向用户和开发者
+- examples：示例代码
+- test：集成测试代码
+
+## API
+
+### remote
+```go
+NewRemotingCommand(code int16, header CustomHeader) *RemotingCommand
+
+// send a request to servers and return until response received.
+SendMessageSync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
+
+SendMessageAsync(ctx context.Context, brokerAddrs, brokerName string, request *SendMessageRequest, msgs []*Message, f func(result *SendResult)) error
+
+SendMessageOneWay(ctx context.Context, brokerAddrs string, request *SendMessageRequest, msgs []*Message) (*SendResult, error)
+```
+
+### common
+All struct needed has been defined in codebase.
+
+```go
+// PullMessage with sync
+SendMessage(topic string, msgs *[]Message) error
+
+// SendMessageAsync send message with batch by async
+SendMessageAsync(topic string, msgs *[]Message, f func(result *SendResult)) error
+
+// PullMessage with sync
+PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequest) (*PullResult, error)
+
+// PullMessageAsync pull message async
+func PullMessageAsync(ctx context.Context, brokerAddrs string, request *PullMessageRequest, f func(result *PullResult)) error
+
+// QueryMaxOffset with specific queueId and topic
+QueryMaxOffset(topic string, queueId int) error
+
+// QueryConsumerOffset with specific queueId and topic of consumerGroup
+QueryConsumerOffset(consumerGroup, topic string, queue int) (int64, error)
+
+// SearchOffsetByTimestamp with specific queueId and topic
+SearchOffsetByTimestamp(topic string, queue int, timestamp int64) (int64, error)
+
+// UpdateConsumerOffset with specific queueId and topic
+UpdateConsumerOffset(consumerGroup, topic string, queue int, offset int64) error
+```
+
+## Road map
+for more details about features: [feature-list](../feature.md)
+
+### Milestone1(due: 2019.3.10)
+
+#### producer
+- [ ] normal message
+- [ ] order message
+
+#### consumer
+- [ ] normal message with pull/push
+- [ ] order message with pull/push
+- [ ] rebalance
+- [ ] offset manager
+
+#### common
+- [ ] API wrapper
+- [ ] connections manager
+- [ ] route
+
+#### remote
+- [ ] serializer
+- [ ] communication
+- [ ] processor
+- [ ] RPC
+
+### Milestone2 (2019.4.12)
+- Transaction Message
+- ACL
+- Message Tracing
+
+## sub project
+- RocketMQ Administration tools: JVM too heavy for command line tools
\ No newline at end of file
diff --git a/docs/zh/rocketmq-protocol_zh.md b/docs/zh/rocketmq-protocol_zh.md
new file mode 100644
index 0000000..818d2e1
--- /dev/null
+++ b/docs/zh/rocketmq-protocol_zh.md
@@ -0,0 +1,117 @@
+# RocketMQ 通信协议
+
+在 RocketMQ 中，`RemotingCommand` 是 RocketMQ 通信的基本对象，Request/Response 最后均被包装成 `RemotingCommand`。一个 `RemotingCommand` 在被序列化后的格式如下：
+
+```
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
++ frame_size | header_length | header_body | body +
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
++ 4bytes | 4bytes | (21 + r_len + e_len) bytes | remain bytes +
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+```
+
+| item | type | description |
+| :-: | :-: | :-: |
+| frame_size | `int32` | 一个 `RemotingCommand` 数据包大小 |
+| header_length | `int32` | 高8位表示数据的序列化方式，余下的表示真实 header 长度 |
+| header_body | `[]byte` | header 的 payload，长度由附带的 `remark` 和 `properties` 决定|
+| body | `[]byte` | 具体 Request/Response 的 payload |
+
+## Header
+
+RocketMQ 的 Header 序列化方式有两种：JSON 和 RocketMQ 私有的序列化方式。JSON 序列化方式不再赘述。具体可以参考 Java `RemotingCommand` 类。
+
+主要介绍 RocketMQ 的私有序列化方式。
+
+在序列化的时候，需要将序列化方式记录进数据包里面，即对 `header_length` 进行编码
+
+```go
+// 编码算法
+
+// 编码后的 header_length
+var header_length int32
+
+// 实际的 header 长度
+var headerDataLen int32
+
+// 序列化方式
+var SerializedType byte
+
+result := make([]byte, 4)
+result[0]|SerializedType
+result[1]|byte((headerDataLen >> 16) & 0xFF)
+result[2]|byte((headerDataLen >> 8) & 0xFF)
+result[3]|byte(headerDataLen & 0xFF)
+binary.Read(result, binary.BigEndian, &header_length)
+
+// 解码算法
+headerDataLen := header_length & 0xFFFFFF
+SerializedType := byte((header_length >> 24) & 0xFF)
+```
+
+### Header Frame
+
+```
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
++ request_code | l_flag | v_flag | opaque | request_flag | r_len | r_body | e_len | e_body +
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
++ 2bytes | 1byte | 2bytes | 4bytes | 4 bytes | 4 bytes | r_len bytes | 4 bytes | e_len bytes +
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+```
+
+| item | type | description |
+| :-: | :-: | :-: |
+| request_code | `int16` | 哪一种 Request 或 ResponseCode，具体类别由 request_flag 决定 |
+| l_flag | `byte` | language 位，用来标识Request来源方的开发语言 |
+| v_flag | `int16` | 版本标记位 |
+| request_flag |`int32`| Header标记位，用来标记该 `RemotingCommand` 的类型和请求方式 |
+| opaque | `int32` | 标识 Request/Response 的 RequestID，Broker 返回的 Response 通过该值和 Client 缓存的 Request 一一对应 |
+| r_len | `int32` | length of remark, remark 是 Request/Response 的附带说明信息，一般在 Response 中用来说明具体的错误原因 |
+| r_body | `[]byte` | payload of remark |
+| e_len | `int32` | length of extended fields，即 properties，一些非标准字段会存储在这里，在 RocketMQ 的各种 feature 中均有广泛应用 |
+| e_body | `int32` | payload of extended fields |
+
+## Body
+
+`body` 是具体的 Request/Response 的数据，在 RocketMQ 中，有许多种 Request/Response。每个类有自己的序列化和反序列方式，由于种类过多，
+这里就不再展开。可以具体参考Java代码中对`CommandCustomHeader`的使用。下面列一些 Client 使用到的 Request 和 Response。
+
+### RequestCode
+
+| item | type | description |
+| :-: | :-: | :-: |
+| SEND_MESSAGE | 10 | 向broker发送消息 |
+| PULL_MESSAGE | 11 | 从broker拉取消息，client的push模式也是通过pull的长轮询来实现的 |
+| TODO... | | |
+
+### ResponseCode
+
+| item | type | description |
+| :-: | :-: | :-: |
+| FLUSH_DISK_TIMEOUT | 10 | broker 存储层刷盘超时 |
+| SLAVE_NOT_AVAILABLE | 11 | slave 节点无法服务 |
+| FLUSH_SLAVE_TIMEOUT | 12 | 数据同步到 slave 超时 |
+| MESSAGE_ILLEGAL | 13 | 消息格式不合格 |
+| SERVICE_NOT_AVAILABLE | 14 | broker 暂时不可用 |
+| VERSION_NOT_SUPPORTED | 15 | 不支持的请求，目前没有看到使用 |
+| NO_PERMISSION | 16 | 对 broker、topic 或 subscription 无访问权限 |
+| TOPIC_EXIST_ALREADY | 18 | topic 已存在，目前没看到使用 |
+| PULL_NOT_FOUND | 19 | 没拉到消息，大多为 offset 错误 |
+| PULL_RETRY_IMMEDIATELY | 20 | 建议 client 立即重新拉取消息 |
+| PULL_OFFSET_MOVED | 21 | offset 太小或太大 |
+| QUERY_NOT_FOUND | 22 | 管理面 Response，TODO |
+| SUBSCRIPTION_PARSE_FAILED | 23 | 订阅数据解析失败 |
+| SUBSCRIPTION_NOT_EXIST | 24 | 订阅不存在 |
+| SUBSCRIPTION_NOT_LATEST | 25 | 订阅数据版本和 request 数据版本不匹配 |
+| SUBSCRIPTION_GROUP_NOT_EXIST | 26 | 订阅组不存在 |
+| FILTER_DATA_NOT_EXIST | 27 | filter 数据不存在 |
+| FILTER_DATA_NOT_LATEST | 28 | filter 数据版本和 request 数据版本不匹配 |
+| TRANSACTION_SHOULD_COMMIT | 200 | 事务 Response，TODO |
+| TRANSACTION_SHOULD_ROLLBACK | 201 | 事务 Response，TODO |
+| TRANSACTION_STATE_UNKNOW | 202 | 事务 Response，TODO | |
+| TRANSACTION_STATE_GROUP_WRONG | 203 | 事务 Response，TODO |
+| NO_BUYER_ID | 204 | 不知道是什么，没看到 broker 端在使用 |
+| NOT_IN_CURRENT_UNIT | 205 | 不知道是什么，没看到 broker 端在使用 |
+| CONSUMER_NOT_ONLINE | 206 | consumer 不在线，控制面 response |
+| CONSUME_MSG_TIMEOUT | 207 | client request 等待 broker 相应超时 |
+| NO_MESSAGE | 208 | 控制面 response，由 client 自己设置，不清楚具体用途 |
diff --git a/errors.go b/errors.go
new file mode 100644
index 0000000..fe9ba33
--- /dev/null
+++ b/errors.go
@@ -0,0 +1,29 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rocketmq
+
+import (
+ "github.com/pkg/errors"
+)
+
+var (
+ ErrRequestTimeout = errors.New("request timeout")
+ ErrMQEmpty = errors.New("MessageQueue is nil")
+ ErrOffset = errors.New("offset < 0")
+ ErrNumbers = errors.New("numbers < 0")
+)
diff --git a/examples/consumer/acl/main.go b/examples/consumer/acl/main.go
new file mode 100644
index 0000000..a6535fd
--- /dev/null
+++ b/examples/consumer/acl/main.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, err := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithCredentials(primitive.Credentials{
+ AccessKey: "RocketMQ",
+ SecretKey: "12345678",
+ }),
+ )
+ if err != nil {
+ fmt.Println("init consumer error: " + err.Error())
+ os.Exit(0)
+ }
+
+ err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("Shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/broadcast/main.go b/examples/consumer/broadcast/main.go
new file mode 100644
index 0000000..29b0b12
--- /dev/null
+++ b/examples/consumer/broadcast/main.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
+ consumer.WithConsumerModel(consumer.BroadCasting),
+ )
+ err := c.Subscribe("min", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("Shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/delay/main.go b/examples/consumer/delay/main.go
new file mode 100644
index 0000000..8cc5c04
--- /dev/null
+++ b/examples/consumer/delay/main.go
@@ -0,0 +1,60 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ )
+ err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+
+ for _, msg := range msgs {
+ t := time.Now().UnixNano()/int64(time.Millisecond) - msg.BornTimestamp
+ fmt.Printf("Receive message[msgId=%s] %d ms later\n", msg.MsgId, t)
+ }
+
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("Shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/interceptor/main.go b/examples/consumer/interceptor/main.go
new file mode 100644
index 0000000..1036c6a
--- /dev/null
+++ b/examples/consumer/interceptor/main.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithConsumerModel(consumer.Clustering),
+ consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
+ consumer.WithInterceptor(UserFistInterceptor(), UserSecondInterceptor()))
+ err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("Shutdown Consumer error: %s", err.Error())
+ }
+}
+
+func UserFistInterceptor() primitive.Interceptor {
+ return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
+ msgCtx, _ := primitive.GetConsumerCtx(ctx)
+ fmt.Printf("msgCtx: %v, mehtod: %s", msgCtx, primitive.GetMethod(ctx))
+
+ msgs := req.([]*primitive.MessageExt)
+ fmt.Printf("user first interceptor before invoke: %v\n", msgs)
+ e := next(ctx, msgs, reply)
+
+ holder := reply.(*consumer.ConsumeResultHolder)
+ fmt.Printf("user first interceptor after invoke: %v, result: %v\n", msgs, holder)
+ return e
+ }
+}
+
+func UserSecondInterceptor() primitive.Interceptor {
+ return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
+ msgs := req.([]*primitive.MessageExt)
+ fmt.Printf("user second interceptor before invoke: %v\n", msgs)
+ e := next(ctx, msgs, reply)
+ holder := reply.(*consumer.ConsumeResultHolder)
+ fmt.Printf("user second interceptor after invoke: %v, result: %v\n", msgs, holder)
+ return e
+ }
+}
diff --git a/examples/consumer/namespace/main.go b/examples/consumer/namespace/main.go
new file mode 100644
index 0000000..e1b9dea
--- /dev/null
+++ b/examples/consumer/namespace/main.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, err := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithCredentials(primitive.Credentials{
+ AccessKey: "RocketMQ",
+ SecretKey: "12345678",
+ }),
+ consumer.WithNamespace("namespace"),
+ )
+ if err != nil {
+ fmt.Println("init consumer error: " + err.Error())
+ os.Exit(0)
+ }
+
+ err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("Shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/orderly/main.go b/examples/consumer/orderly/main.go
new file mode 100644
index 0000000..9e7a810
--- /dev/null
+++ b/examples/consumer/orderly/main.go
@@ -0,0 +1,60 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithConsumerModel(consumer.Clustering),
+ consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
+ consumer.WithConsumerOrder(true),
+ )
+ err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
+ fmt.Printf("orderly context: %v\n", orderlyCtx)
+ fmt.Printf("subscribe orderly callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("Shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/pull/main.go b/examples/consumer/pull/main.go
new file mode 100644
index 0000000..d740b15
--- /dev/null
+++ b/examples/consumer/pull/main.go
@@ -0,0 +1,71 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+func main() {
+ c, err := rocketmq.NewPullConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ )
+ if err != nil {
+ rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
+ }
+ err = c.Start()
+ if err != nil {
+ rlog.Fatal(fmt.Sprintf("fail to new pullConsumer: %s", err), nil)
+ }
+
+ ctx := context.Background()
+ queue := primitive.MessageQueue{
+ Topic: "TopicTest",
+ BrokerName: "", // replace with your broker name. otherwise, pull will failed.
+ QueueId: 0,
+ }
+
+ offset := int64(0)
+ for {
+ resp, err := c.PullFrom(ctx, queue, offset, 10)
+ if err != nil {
+ if err == rocketmq.ErrRequestTimeout {
+ fmt.Printf("timeout \n")
+ time.Sleep(1 * time.Second)
+ continue
+ }
+ fmt.Printf("unexpectable err: %v \n", err)
+ return
+ }
+ if resp.Status == primitive.PullFound {
+ fmt.Printf("pull message success. nextOffset: %d \n", resp.NextBeginOffset)
+ for _, msg := range resp.GetMessageExts() {
+ fmt.Printf("pull msg: %v \n", msg)
+ }
+ }
+ offset = resp.NextBeginOffset
+ }
+}
diff --git a/examples/consumer/retry/concurrent/main.go b/examples/consumer/retry/concurrent/main.go
new file mode 100644
index 0000000..5fcf489
--- /dev/null
+++ b/examples/consumer/retry/concurrent/main.go
@@ -0,0 +1,81 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+// use concurrent consumer model, when Subscribe function return consumer.ConsumeRetryLater, the message will be
+// send to RocketMQ retry topic. we could set DelayLevelWhenNextConsume in ConsumeConcurrentlyContext, which used to
+// indicate the delay of message re-send to origin topic from retry topic.
+//
+// in this example, we always set DelayLevelWhenNextConsume=1, means that the message will be sent to origin topic after
+// 1s. in case of the unlimited retry, we will return consumer.ConsumeSuccess after ReconsumeTimes > 5
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithConsumerModel(consumer.Clustering),
+ )
+
+ // The DelayLevel specify the waiting time that before next reconsume,
+ // and it range is from 1 to 18 now.
+ //
+ // The time of each level is the value of indexing of {level-1} in [1s, 5s, 10s, 30s,
+ // 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 1h, 2h]
+ delayLevel := 1
+ err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback len: %d \n", len(msgs))
+
+ concurrentCtx, _ := primitive.GetConcurrentlyCtx(ctx)
+ concurrentCtx.DelayLevelWhenNextConsume = delayLevel // only run when return consumer.ConsumeRetryLater
+
+ for _, msg := range msgs {
+ if msg.ReconsumeTimes > 5 {
+ fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
+ return consumer.ConsumeSuccess, nil
+ } else {
+ fmt.Printf("subscribe callback: %v \n", msg)
+ }
+ }
+ return consumer.ConsumeRetryLater, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("shundown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/retry/order/main.go b/examples/consumer/retry/order/main.go
new file mode 100644
index 0000000..4ec05e7
--- /dev/null
+++ b/examples/consumer/retry/order/main.go
@@ -0,0 +1,76 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * use orderly consumer model, when Subscribe function return consumer.SuspendCurrentQueueAMoment, it will be re-send to
+ * local msg queue for later consume if msg.ReconsumeTimes < MaxReconsumeTimes, otherwise, it will be send to rocketmq
+ * DLQ topic, we should manually resolve the msg.
+ */
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithConsumerModel(consumer.Clustering),
+ consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset),
+ consumer.WithConsumerOrder(true),
+ consumer.WithMaxReconsumeTimes(5),
+ )
+
+ err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ orderlyCtx, _ := primitive.GetOrderlyCtx(ctx)
+ fmt.Printf("orderly context: %v\n", orderlyCtx)
+ fmt.Printf("subscribe orderly callback len: %d \n", len(msgs))
+
+ for _, msg := range msgs {
+ if msg.ReconsumeTimes > 5 {
+ fmt.Printf("msg ReconsumeTimes > 5. msg: %v", msg)
+ } else {
+ fmt.Printf("subscribe orderly callback: %v \n", msg)
+ }
+ }
+ return consumer.SuspendCurrentQueueAMoment, nil
+
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("shundown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/simple/main.go b/examples/consumer/simple/main.go
new file mode 100644
index 0000000..7d1a0b7
--- /dev/null
+++ b/examples/consumer/simple/main.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ )
+ err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ for i := range msgs {
+ fmt.Printf("subscribe callback: %v \n", msgs[i])
+ }
+
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/strategy/main.go b/examples/consumer/strategy/main.go
new file mode 100644
index 0000000..502524c
--- /dev/null
+++ b/examples/consumer/strategy/main.go
@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithStrategy(consumer.AllocateByAveragely),
+ )
+ err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/tag/main.go b/examples/consumer/tag/main.go
new file mode 100644
index 0000000..0532971
--- /dev/null
+++ b/examples/consumer/tag/main.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ )
+ selector := consumer.MessageSelector{
+ Type: consumer.TAG,
+ Expression: "TagA || TagC",
+ }
+ err := c.Subscribe("TopicTest", selector, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/consumer/trace/main.go b/examples/consumer/trace/main.go
new file mode 100644
index 0000000..35c4926
--- /dev/null
+++ b/examples/consumer/trace/main.go
@@ -0,0 +1,63 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/consumer"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func main() {
+ namesrvs := []string{"127.0.0.1:9876"}
+ traceCfg := &primitive.TraceConfig{
+ Access: primitive.Local,
+ NamesrvAddrs: namesrvs,
+ }
+
+ c, _ := rocketmq.NewPushConsumer(
+ consumer.WithGroupName("testGroup"),
+ consumer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ consumer.WithTrace(traceCfg),
+ )
+ err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context,
+ msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) {
+ fmt.Printf("subscribe callback: %v \n", msgs)
+ return consumer.ConsumeSuccess, nil
+ })
+ if err != nil {
+ fmt.Println(err.Error())
+ }
+ // Note: start after subscribe
+ err = c.Start()
+ if err != nil {
+ fmt.Println(err.Error())
+ os.Exit(-1)
+
+ }
+ time.Sleep(time.Hour)
+ err = c.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown Consumer error: %s", err.Error())
+ }
+}
diff --git a/examples/orderly_push_consumer.go b/examples/orderly_push_consumer.go
deleted file mode 100644
index cc457a6..0000000
--- a/examples/orderly_push_consumer.go
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/core"
- "math/rand"
- "sync/atomic"
-)
-
-// Change to main if you want to run it directly
-func main3() {
- pConfig := &rocketmq.PushConsumerConfig{
- ClientConfig: rocketmq.ClientConfig{
- GroupID: "GID_XXXXXXXXXXXX",
- NameServer: "http://XXXXXXXXXXXXXXXXXX:80",
- Credentials: &rocketmq.SessionCredentials{
- AccessKey: "Your Access Key",
- SecretKey: "Your Secret Key",
- Channel: "ALIYUN/OtherChannel",
- },
- },
- Model: rocketmq.Clustering,
- ConsumerModel: rocketmq.Orderly,
- }
- consumeWithOrderly(pConfig)
-}
-
-func consumeWithOrderly(config *rocketmq.PushConsumerConfig) {
-
- consumer, err := rocketmq.NewPushConsumer(config)
- if err != nil {
- println("create Consumer failed, error:", err)
- return
- }
-
- ch := make(chan interface{})
- var count = (int64)(1000000)
- // ********************************************
- // MUST subscribe topic before consumer started.
- // *********************************************
- consumer.Subscribe("YourOrderlyTopicXXXXXXXX", "*", func(msg *rocketmq.MessageExt) rocketmq.ConsumeStatus {
- fmt.Printf("A message received, MessageID:%s, Body:%s \n", msg.MessageID, msg.Body)
- if atomic.AddInt64(&count, -1) <= 0 {
- ch <- "quit"
- }
- if 0 == rand.Int()%7 {
- fmt.Printf("Consumer Later, MessageID:%s \n", msg.MessageID)
- return rocketmq.ReConsumeLater
- }
- return rocketmq.ConsumeSuccess
- })
-
- err = consumer.Start()
- if err != nil {
- println("consumer start failed,", err)
- return
- }
-
- fmt.Printf("consumer: %s started...\n", consumer)
- <-ch
- err = consumer.Shutdown()
- if err != nil {
- println("consumer shutdown failed")
- return
- }
- println("consumer has shutdown.")
-}
diff --git a/examples/producer.go b/examples/producer.go
deleted file mode 100644
index ad08c12..0000000
--- a/examples/producer.go
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/core"
-)
-
-// Change to main if you want to run it directly
-func main0() {
- pConfig := &rocketmq.ProducerConfig{
- ClientConfig: rocketmq.ClientConfig{
- GroupID: "GID_XXXXXXXXXXXX",
- NameServer: "http://XXXXXXXXXXXXXXXXXX:80",
- Credentials: &rocketmq.SessionCredentials{
- AccessKey: "Your Access Key",
- SecretKey: "Your Secret Key",
- Channel: "ALIYUN/OtherChannel",
- },
- },
- //Set to Common Producer as default.
- ProducerModel: rocketmq.CommonProducer,
- }
- sendMessage(pConfig)
-}
-func sendMessage(config *rocketmq.ProducerConfig) {
- producer, err := rocketmq.NewProducer(config)
-
- if err != nil {
- fmt.Println("create common producer failed, error:", err)
- return
- }
-
- err = producer.Start()
- if err != nil {
- fmt.Println("start common producer error", err)
- return
- }
- defer producer.Shutdown()
-
- fmt.Printf("Common producer: %s started... \n", producer)
- for i := 0; i < 10; i++ {
- msg := fmt.Sprintf("%s-%d", "Hello,Common MQ Message-", i)
- result, err := producer.SendMessageSync(&rocketmq.Message{Topic: "YourTopicXXXXXXXX", Body: msg})
- if err != nil {
- fmt.Println("Error:", err)
- }
- fmt.Printf("send message: %s result: %s\n", msg, result)
- }
- fmt.Println("shutdown common producer.")
-}
diff --git a/examples/producer/acl/main.go b/examples/producer/acl/main.go
new file mode 100644
index 0000000..38d61dc
--- /dev/null
+++ b/examples/producer/acl/main.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package main implements a producer with user custom interceptor.
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+func main() {
+ p, err := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ producer.WithCredentials(primitive.Credentials{
+ AccessKey: "RocketMQ",
+ SecretKey: "12345678",
+ }),
+ )
+
+ if err != nil {
+ fmt.Println("init producer error: " + err.Error())
+ os.Exit(0)
+ }
+
+ err = p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ for i := 0; i < 100000; i++ {
+ res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
+ []byte("Hello RocketMQ Go Client!")))
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/async/main.go b/examples/producer/async/main.go
new file mode 100644
index 0000000..aa73881
--- /dev/null
+++ b/examples/producer/async/main.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "sync"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+// Package main implements a async producer to send message.
+func main() {
+ p, _ := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ producer.WithQueueSelector(producer.NewManualQueueSelector()))
+
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ var wg sync.WaitGroup
+ for i := 0; i < 10; i++ {
+ wg.Add(1)
+ err := p.SendAsync(context.Background(),
+ func(ctx context.Context, result *primitive.SendResult, e error) {
+ if e != nil {
+ fmt.Printf("receive message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", result.String())
+ }
+ wg.Done()
+ }, primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!")))
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ }
+ }
+ wg.Wait()
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/batch/main.go b/examples/producer/batch/main.go
new file mode 100644
index 0000000..d807daf
--- /dev/null
+++ b/examples/producer/batch/main.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strconv"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+func main() {
+ p, _ := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ )
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ var msgs []*primitive.Message
+ for i := 0; i < 10; i++ {
+ msgs = append(msgs, primitive.NewMessage("test",
+ []byte("Hello RocketMQ Go Client! num: "+strconv.Itoa(i))))
+ }
+
+ res, err := p.SendSync(context.Background(), msgs...)
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/delay/main.go b/examples/producer/delay/main.go
new file mode 100644
index 0000000..465e97c
--- /dev/null
+++ b/examples/producer/delay/main.go
@@ -0,0 +1,55 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+func main() {
+ p, _ := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ )
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ for i := 0; i < 10; i++ {
+ msg := primitive.NewMessage("test", []byte("Hello RocketMQ Go Client!"))
+ msg.WithDelayTimeLevel(3)
+ res, err := p.SendSync(context.Background(), msg)
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/interceptor/main.go b/examples/producer/interceptor/main.go
new file mode 100644
index 0000000..47ea1fb
--- /dev/null
+++ b/examples/producer/interceptor/main.go
@@ -0,0 +1,74 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package main implements a producer with user custom interceptor.
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+func main() {
+ p, _ := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ producer.WithInterceptor(UserFirstInterceptor(), UserSecondInterceptor()),
+ )
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ for i := 0; i < 10; i++ {
+ res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
+ []byte("Hello RocketMQ Go Client!")))
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
+
+func UserFirstInterceptor() primitive.Interceptor {
+ return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
+ fmt.Printf("user first interceptor before invoke: req:%v\n", req)
+ err := next(ctx, req, reply)
+ fmt.Printf("user first interceptor after invoke: req: %v, reply: %v \n", req, reply)
+ return err
+ }
+}
+
+func UserSecondInterceptor() primitive.Interceptor {
+ return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
+ fmt.Printf("user second interceptor before invoke: req: %v\n", req)
+ err := next(ctx, req, reply)
+ fmt.Printf("user second interceptor after invoke: req: %v, reply: %v \n", req, reply)
+ return err
+ }
+}
diff --git a/examples/producer/namespace/main.go b/examples/producer/namespace/main.go
new file mode 100644
index 0000000..524bd32
--- /dev/null
+++ b/examples/producer/namespace/main.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package main implements a producer with user custom interceptor.
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+func main() {
+ p, err := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ producer.WithCredentials(primitive.Credentials{
+ AccessKey: "RocketMQ",
+ SecretKey: "12345678",
+ }),
+ producer.WithNamespace("namespace"),
+ )
+
+ if err != nil {
+ fmt.Println("init producer error: " + err.Error())
+ os.Exit(0)
+ }
+
+ err = p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ for i := 0; i < 100000; i++ {
+ res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
+ []byte("Hello RocketMQ Go Client!")))
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/simple/main.go b/examples/producer/simple/main.go
new file mode 100644
index 0000000..8ac4421
--- /dev/null
+++ b/examples/producer/simple/main.go
@@ -0,0 +1,61 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strconv"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+// Package main implements a simple producer to send message.
+func main() {
+ p, _ := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ )
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ topic := "test"
+
+ for i := 0; i < 10; i++ {
+ msg := &primitive.Message{
+ Topic: topic,
+ Body: []byte("Hello RocketMQ Go Client! " + strconv.Itoa(i)),
+ }
+ res, err := p.SendSync(context.Background(), msg)
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/tag/main.go b/examples/producer/tag/main.go
new file mode 100644
index 0000000..2bcd51b
--- /dev/null
+++ b/examples/producer/tag/main.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+func main() {
+ p, _ := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ )
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ tags := []string{"TagA", "TagB", "TagC"}
+ for i := 0; i < 3; i++ {
+ tag := tags[i%3]
+ msg := primitive.NewMessage("test",
+ []byte("Hello RocketMQ Go Client!"))
+ msg.WithTag(tag)
+
+ res, err := p.SendSync(context.Background(), msg)
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/trace/main.go b/examples/producer/trace/main.go
new file mode 100644
index 0000000..5f2c9b3
--- /dev/null
+++ b/examples/producer/trace/main.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+func main() {
+ namesrvs := []string{"127.0.0.1:9876"}
+ traceCfg := &primitive.TraceConfig{
+ Access: primitive.Local,
+ NamesrvAddrs: namesrvs,
+ }
+
+ p, _ := rocketmq.NewProducer(
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(2),
+ producer.WithTrace(traceCfg))
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s", err.Error())
+ os.Exit(1)
+ }
+ for i := 0; i < 1; i++ {
+ res, err := p.SendSync(context.Background(), primitive.NewMessage("test",
+ []byte("Hello RocketMQ Go Client!")))
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+
+ time.Sleep(10 * time.Second)
+
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer/transaction/main.go b/examples/producer/transaction/main.go
new file mode 100644
index 0000000..dde39a9
--- /dev/null
+++ b/examples/producer/transaction/main.go
@@ -0,0 +1,106 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/producer"
+)
+
+type DemoListener struct {
+ localTrans *sync.Map
+ transactionIndex int32
+}
+
+func NewDemoListener() *DemoListener {
+ return &DemoListener{
+ localTrans: new(sync.Map),
+ }
+}
+
+func (dl *DemoListener) ExecuteLocalTransaction(msg *primitive.Message) primitive.LocalTransactionState {
+ nextIndex := atomic.AddInt32(&dl.transactionIndex, 1)
+ fmt.Printf("nextIndex: %v for transactionID: %v\n", nextIndex, msg.TransactionId)
+ status := nextIndex % 3
+ dl.localTrans.Store(msg.TransactionId, primitive.LocalTransactionState(status+1))
+
+ fmt.Printf("dl")
+ return primitive.UnknowState
+}
+
+func (dl *DemoListener) CheckLocalTransaction(msg *primitive.MessageExt) primitive.LocalTransactionState {
+ fmt.Printf("%v msg transactionID : %v\n", time.Now(), msg.TransactionId)
+ v, existed := dl.localTrans.Load(msg.TransactionId)
+ if !existed {
+ fmt.Printf("unknow msg: %v, return Commit", msg)
+ return primitive.CommitMessageState
+ }
+ state := v.(primitive.LocalTransactionState)
+ switch state {
+ case 1:
+ fmt.Printf("checkLocalTransaction COMMIT_MESSAGE: %v\n", msg)
+ return primitive.CommitMessageState
+ case 2:
+ fmt.Printf("checkLocalTransaction ROLLBACK_MESSAGE: %v\n", msg)
+ return primitive.RollbackMessageState
+ case 3:
+ fmt.Printf("checkLocalTransaction unknow: %v\n", msg)
+ return primitive.UnknowState
+ default:
+ fmt.Printf("checkLocalTransaction default COMMIT_MESSAGE: %v\n", msg)
+ return primitive.CommitMessageState
+ }
+}
+
+func main() {
+ p, _ := rocketmq.NewTransactionProducer(
+ NewDemoListener(),
+ producer.WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ producer.WithRetry(1),
+ )
+ err := p.Start()
+ if err != nil {
+ fmt.Printf("start producer error: %s\n", err.Error())
+ os.Exit(1)
+ }
+
+ for i := 0; i < 10; i++ {
+ res, err := p.SendMessageInTransaction(context.Background(),
+ primitive.NewMessage("TopicTest5", []byte("Hello RocketMQ again "+strconv.Itoa(i))))
+
+ if err != nil {
+ fmt.Printf("send message error: %s\n", err)
+ } else {
+ fmt.Printf("send message success: result=%s\n", res.String())
+ }
+ }
+ time.Sleep(5 * time.Minute)
+ err = p.Shutdown()
+ if err != nil {
+ fmt.Printf("shutdown producer error: %s", err.Error())
+ }
+}
diff --git a/examples/producer_orderly.go b/examples/producer_orderly.go
deleted file mode 100644
index e379a93..0000000
--- a/examples/producer_orderly.go
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/core"
- "time"
-)
-
-// Change to main if you want to run it directly
-func main2() {
- pConfig := &rocketmq.ProducerConfig{
- ClientConfig: rocketmq.ClientConfig{
- GroupID: "GID_XXXXXXXXXXXX",
- NameServer: "http://XXXXXXXXXXXXXXXXXX:80",
- Credentials: &rocketmq.SessionCredentials{
- AccessKey: "Your Access Key",
- SecretKey: "Your Secret Key",
- Channel: "ALIYUN/OtherChannel",
- },
- },
- ProducerModel: rocketmq.OrderlyProducer,
- }
- sendMessageOrderlyByShardingKey(pConfig)
-}
-func sendMessageOrderlyByShardingKey(config *rocketmq.ProducerConfig) {
- producer, err := rocketmq.NewProducer(config)
- if err != nil {
- fmt.Println("create Producer failed, error:", err)
- return
- }
-
- producer.Start()
- defer producer.Shutdown()
- for i := 0; i < 1000; i++ {
- msg := fmt.Sprintf("%s-%d", "Hello Lite Orderly Message", i)
- r, err := producer.SendMessageOrderlyByShardingKey(
- &rocketmq.Message{Topic: "YourOrderLyTopicXXXXXXXX", Body: msg}, "ShardingKey" /*orderID*/)
- if err != nil {
- println("Send Orderly Message Error:", err)
- }
- fmt.Printf("send orderly message result:%+v\n", r)
- time.Sleep(time.Duration(1) * time.Second)
- }
-
-}
diff --git a/examples/push_consumer.go b/examples/push_consumer.go
deleted file mode 100644
index 98a1ddf..0000000
--- a/examples/push_consumer.go
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/core"
- "sync/atomic"
-)
-
-// Change to main if you want to run it directly
-func main1() {
- pConfig := &rocketmq.PushConsumerConfig{
- ClientConfig: rocketmq.ClientConfig{
- GroupID: "GID_XXXXXXXXXXXX",
- NameServer: "http://XXXXXXXXXXXXXXXXXX:80",
- Credentials: &rocketmq.SessionCredentials{
- AccessKey: "Your Access Key",
- SecretKey: "Your Secret Key",
- Channel: "ALIYUN/OtherChannel",
- },
- },
- Model: rocketmq.Clustering,
- ConsumerModel: rocketmq.CoCurrently,
- }
- consumeWithPush(pConfig)
-}
-func consumeWithPush(config *rocketmq.PushConsumerConfig) {
-
- consumer, err := rocketmq.NewPushConsumer(config)
- if err != nil {
- println("create Consumer failed, error:", err)
- return
- }
-
- ch := make(chan interface{})
- var count = (int64)(1000000)
- // ********************************************
- // MUST subscribe topic before consumer started.
- // *********************************************
- consumer.Subscribe("YourTopicXXXXXXXX", "*", func(msg *rocketmq.MessageExt) rocketmq.ConsumeStatus {
- fmt.Printf("A message received, MessageID:%s, Body:%s \n", msg.MessageID, msg.Body)
- if atomic.AddInt64(&count, -1) <= 0 {
- ch <- "quit"
- }
- return rocketmq.ConsumeSuccess
- })
-
- err = consumer.Start()
- if err != nil {
- println("consumer start failed,", err)
- return
- }
-
- fmt.Printf("consumer: %s started...\n", consumer)
- <-ch
- err = consumer.Shutdown()
- if err != nil {
- println("consumer shutdown failed")
- return
- }
- println("consumer has shutdown.")
-}
diff --git a/examples/transaction_producer.go b/examples/transaction_producer.go
deleted file mode 100644
index 762c4c8..0000000
--- a/examples/transaction_producer.go
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
- "fmt"
- "github.com/apache/rocketmq-client-go/core"
- "time"
-)
-
-// Change to main if you want to run it directly
-func main4() {
- pConfig := &rocketmq.ProducerConfig{
- ClientConfig: rocketmq.ClientConfig{
- GroupID: "GID_XXXXXXXXXXXX",
- NameServer: "http://XXXXXXXXXXXXXXXXXX:80",
- Credentials: &rocketmq.SessionCredentials{
- AccessKey: "Your Access Key",
- SecretKey: "Your Secret Key",
- Channel: "ALIYUN/OtherChannel",
- },
- },
- //Set to Trans Producer as default.
- ProducerModel: rocketmq.TransProducer,
- }
- sendTransactionMessage(pConfig)
-}
-
-type myTransactionLocalListener struct {
-}
-
-func (l *myTransactionLocalListener) Execute(m *rocketmq.Message, arg interface{}) rocketmq.TransactionStatus {
- return rocketmq.UnknownTransaction
-}
-func (l *myTransactionLocalListener) Check(m *rocketmq.MessageExt, arg interface{}) rocketmq.TransactionStatus {
- return rocketmq.CommitTransaction
-}
-func sendTransactionMessage(config *rocketmq.ProducerConfig) {
- listener := &myTransactionLocalListener{}
- producer, err := rocketmq.NewTransactionProducer(config, listener, nil)
-
- if err != nil {
- fmt.Println("create Transaction producer failed, error:", err)
- return
- }
-
- err = producer.Start()
- if err != nil {
- fmt.Println("start Transaction producer error", err)
- return
- }
- defer producer.Shutdown()
-
- fmt.Printf("Transaction producer: %s started... \n", producer)
- for i := 0; i < 10; i++ {
- msg := fmt.Sprintf("%s-%d", "Hello,Transaction MQ Message-", i)
- result, err := producer.SendMessageTransaction(&rocketmq.Message{Topic: "YourTopicXXXXXXXX", Body: msg}, nil)
- if err != nil {
- fmt.Println("Error:", err)
- }
- fmt.Printf("send message: %s result: %s\n", msg, result)
- }
- time.Sleep(time.Duration(1) * time.Minute)
- fmt.Println("shutdown Transaction producer.")
-}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..8388959
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,20 @@
+module github.com/apache/rocketmq-client-go/v2
+
+require (
+ github.com/emirpasic/gods v1.12.0
+ github.com/golang/mock v1.3.1
+ github.com/json-iterator/go v1.1.9
+ github.com/pkg/errors v0.8.1
+ github.com/sirupsen/logrus v1.4.1
+ github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945
+ github.com/stretchr/testify v1.3.0
+ github.com/tidwall/gjson v1.2.1
+ github.com/tidwall/match v1.0.1 // indirect
+ github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 // indirect
+ go.uber.org/atomic v1.5.1
+ stathat.com/c/consistent v1.0.0
+)
+
+replace stathat.com/c/consistent v1.0.0 => github.com/stathat/consistent v1.0.0
+
+go 1.13
diff --git a/go.sum b/go.sum
new file mode 100644
index 0000000..3b724db
--- /dev/null
+++ b/go.sum
@@ -0,0 +1,63 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 h1:N8Bg45zpk/UcpNGnfJt2y/3lRWASHNTUET8owPYCgYI=
+github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U=
+github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tidwall/gjson v1.2.1 h1:j0efZLrZUvNerEf6xqoi0NjWMK5YlLrR7Guo/dxY174=
+github.com/tidwall/gjson v1.2.1/go.mod h1:c/nTNbUr0E0OrXEhq1pwa8iEgc2DOt4ZZqAt1HtCkPA=
+github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
+github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
+github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65 h1:rQ229MBgvW68s1/g6f1/63TgYwYxfF4E+bi/KC19P8g=
+github.com/tidwall/pretty v0.0.0-20190325153808-1166b9ac2b65/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+go.uber.org/atomic v1.5.1 h1:rsqfU5vBkVknbhUGbAUwQKR2H4ItV8tjJ+6kJX4cxHM=
+go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c h1:IGkKhmfzcztjm6gYkykvu/NiS8kaqbCWAEWWAyf8J5U=
+golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/internal/callback.go b/internal/callback.go
new file mode 100644
index 0000000..fea11b4
--- /dev/null
+++ b/internal/callback.go
@@ -0,0 +1,31 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "net"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+// remotingClient callback TransactionProducer
+type CheckTransactionStateCallback struct {
+ Addr net.Addr
+ Msg *primitive.MessageExt
+ Header CheckTransactionStateRequestHeader
+}
diff --git a/internal/client.go b/internal/client.go
new file mode 100644
index 0000000..37d7ea8
--- /dev/null
+++ b/internal/client.go
@@ -0,0 +1,778 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+const (
+ clientVersion = "v2.0.0"
+ defaultTraceRegionID = "DefaultRegion"
+
+ // tracing message switch
+ _TranceOff = "false"
+
+ // Pulling topic information interval from the named server
+ _PullNameServerInterval = 30 * time.Second
+
+ // Sending heart beat interval to all broker
+ _HeartbeatBrokerInterval = 30 * time.Second
+
+ // Offset persistent interval for consumer
+ _PersistOffsetInterval = 5 * time.Second
+
+ // Rebalance interval
+ _RebalanceInterval = 20 * time.Second
+)
+
+var (
+ ErrServiceState = errors.New("service close is not running, please check")
+
+ _VIPChannelEnable = false
+)
+
+func init() {
+ if os.Getenv("com.rocketmq.sendMessageWithVIPChannel") != "" {
+ value, err := strconv.ParseBool(os.Getenv("com.rocketmq.sendMessageWithVIPChannel"))
+ if err == nil {
+ _VIPChannelEnable = value
+ }
+ }
+}
+
+type InnerProducer interface {
+ PublishTopicList() []string
+ UpdateTopicPublishInfo(topic string, info *TopicPublishInfo)
+ IsPublishTopicNeedUpdate(topic string) bool
+ IsUnitMode() bool
+}
+
+type InnerConsumer interface {
+ PersistConsumerOffset() error
+ UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue)
+ IsSubscribeTopicNeedUpdate(topic string) bool
+ SubscriptionDataList() []*SubscriptionData
+ Rebalance()
+ IsUnitMode() bool
+ GetConsumerRunningInfo() *ConsumerRunningInfo
+ GetcType() string
+ GetModel() string
+ GetWhere() string
+}
+
+func DefaultClientOptions() ClientOptions {
+ opts := ClientOptions{
+ InstanceName: "DEFAULT",
+ RetryTimes: 3,
+ ClientIP: utils.LocalIP,
+ }
+ return opts
+}
+
+type ClientOptions struct {
+ GroupName string
+ NameServerAddrs primitive.NamesrvAddr
+ Namesrv *namesrvs
+ ClientIP string
+ InstanceName string
+ UnitMode bool
+ UnitName string
+ VIPChannelEnabled bool
+ RetryTimes int
+ Interceptors []primitive.Interceptor
+ Credentials primitive.Credentials
+ Namespace string
+ Resolver primitive.NsResolver
+}
+
+func (opt *ClientOptions) ChangeInstanceNameToPID() {
+ if opt.InstanceName == "DEFAULT" {
+ opt.InstanceName = strconv.Itoa(os.Getpid())
+ }
+}
+
+func (opt *ClientOptions) String() string {
+ return fmt.Sprintf("ClientOption [ClientIP=%s, InstanceName=%s, "+
+ "UnitMode=%v, UnitName=%s, VIPChannelEnabled=%v]", opt.ClientIP,
+ opt.InstanceName, opt.UnitMode, opt.UnitName, opt.VIPChannelEnabled)
+}
+
+//go:generate mockgen -source client.go -destination mock_client.go -self_package github.com/apache/rocketmq-client-go/v2/internal --package internal RMQClient
+type RMQClient interface {
+ Start()
+ Shutdown()
+
+ ClientID() string
+
+ RegisterProducer(group string, producer InnerProducer)
+ UnregisterProducer(group string)
+ InvokeSync(ctx context.Context, addr string, request *remote.RemotingCommand,
+ timeoutMillis time.Duration) (*remote.RemotingCommand, error)
+ InvokeAsync(ctx context.Context, addr string, request *remote.RemotingCommand,
+ f func(*remote.RemotingCommand, error)) error
+ InvokeOneWay(ctx context.Context, addr string, request *remote.RemotingCommand,
+ timeoutMillis time.Duration) error
+ CheckClientInBroker()
+ SendHeartbeatToAllBrokerWithLock()
+ UpdateTopicRouteInfo()
+
+ ProcessSendResponse(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) error
+
+ RegisterConsumer(group string, consumer InnerConsumer) error
+ UnregisterConsumer(group string)
+ PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequestHeader) (*primitive.PullResult, error)
+ RebalanceImmediately()
+ UpdatePublishInfo(topic string, data *TopicRouteData, changed bool)
+}
+
+var _ RMQClient = new(rmqClient)
+
+type rmqClient struct {
+ option ClientOptions
+ // group -> InnerProducer
+ producerMap sync.Map
+
+ // group -> InnerConsumer
+ consumerMap sync.Map
+ once sync.Once
+
+ remoteClient remote.RemotingClient
+ hbMutex sync.Mutex
+ close bool
+ rbMutex sync.Mutex
+ namesrvs *namesrvs
+ done chan struct{}
+ shutdownOnce sync.Once
+}
+
+var clientMap sync.Map
+
+func GetOrNewRocketMQClient(option ClientOptions, callbackCh chan interface{}) RMQClient {
+ client := &rmqClient{
+ option: option,
+ remoteClient: remote.NewRemotingClient(),
+ namesrvs: option.Namesrv,
+ done: make(chan struct{}),
+ }
+ actual, loaded := clientMap.LoadOrStore(client.ClientID(), client)
+ if !loaded {
+ client.remoteClient.RegisterRequestFunc(ReqNotifyConsumerIdsChanged, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
+ rlog.Info("receive broker's notification to consumer group", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: req.ExtFields["consumerGroup"],
+ })
+ client.RebalanceImmediately()
+ return nil
+ })
+ client.remoteClient.RegisterRequestFunc(ReqCheckTransactionState, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
+ header := new(CheckTransactionStateRequestHeader)
+ header.Decode(req.ExtFields)
+ msgExts := primitive.DecodeMessage(req.Body)
+ if len(msgExts) == 0 {
+ rlog.Warning("checkTransactionState, decode message failed", nil)
+ return nil
+ }
+ msgExt := msgExts[0]
+ // TODO: add namespace support
+ transactionID := msgExt.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
+ if len(transactionID) > 0 {
+ msgExt.TransactionId = transactionID
+ }
+ group := msgExt.GetProperty(primitive.PropertyProducerGroup)
+ if group == "" {
+ rlog.Warning("checkTransactionState, pick producer group failed", nil)
+ return nil
+ }
+ if option.GroupName != group {
+ rlog.Warning("producer group is not equal", nil)
+ return nil
+ }
+ callback := &CheckTransactionStateCallback{
+ Addr: addr,
+ Msg: msgExt,
+ Header: *header,
+ }
+ callbackCh <- callback
+ return nil
+ })
+
+ client.remoteClient.RegisterRequestFunc(ReqGetConsumerRunningInfo, func(req *remote.RemotingCommand, addr net.Addr) *remote.RemotingCommand {
+ rlog.Info("receive get consumer running info request...", nil)
+ header := new(GetConsumerRunningInfoHeader)
+ header.Decode(req.ExtFields)
+ val, exist := clientMap.Load(header.clientID)
+ res := remote.NewRemotingCommand(ResError, nil, nil)
+ if !exist {
+ res.Remark = fmt.Sprintf("Can't find specified client instance of: %s", header.clientID)
+ } else {
+ cli, ok := val.(*rmqClient)
+ var runningInfo *ConsumerRunningInfo
+ if ok {
+ runningInfo = cli.getConsumerRunningInfo(header.consumerGroup)
+ }
+ if runningInfo != nil {
+ res.Code = ResSuccess
+ data, err := runningInfo.Encode()
+ if err != nil {
+ res.Remark = fmt.Sprintf("json marshal error: %s", err.Error())
+ } else {
+ res.Body = data
+ }
+ } else {
+ res.Remark = "there is unexpected error when get running info, please check log"
+ }
+ }
+ return res
+ })
+ }
+ return actual.(*rmqClient)
+}
+
+func (c *rmqClient) Start() {
+ //ctx, cancel := context.WithCancel(context.Background())
+ //c.cancel = cancel
+ c.once.Do(func() {
+ if !c.option.Credentials.IsEmpty() {
+ c.remoteClient.RegisterInterceptor(remote.ACLInterceptor(c.option.Credentials))
+ }
+ go primitive.WithRecover(func() {
+ op := func() {
+ c.namesrvs.UpdateNameServerAddress()
+ }
+ time.Sleep(10 * time.Second)
+ op()
+
+ ticker := time.NewTicker(2 * time.Minute)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ op()
+ case <-c.done:
+ rlog.Info("The RMQClient stopping update name server domain info.", map[string]interface{}{
+ "clientID": c.ClientID(),
+ })
+ return
+ }
+ }
+ })
+
+ // schedule update route info
+ go primitive.WithRecover(func() {
+ // delay
+ op := func() {
+ c.UpdateTopicRouteInfo()
+ }
+ time.Sleep(10 * time.Millisecond)
+ op()
+
+ ticker := time.NewTicker(_PullNameServerInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ op()
+ case <-c.done:
+ rlog.Info("The RMQClient stopping update topic route info.", map[string]interface{}{
+ "clientID": c.ClientID(),
+ })
+ return
+ }
+ }
+ })
+
+ go primitive.WithRecover(func() {
+ op := func() {
+ c.namesrvs.cleanOfflineBroker()
+ c.SendHeartbeatToAllBrokerWithLock()
+ }
+
+ time.Sleep(time.Second)
+ op()
+
+ ticker := time.NewTicker(_HeartbeatBrokerInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ op()
+ case <-c.done:
+ rlog.Info("The RMQClient stopping clean off line broker and heart beat", map[string]interface{}{
+ "clientID": c.ClientID(),
+ })
+ return
+ }
+ }
+ })
+
+ // schedule persist offset
+ go primitive.WithRecover(func() {
+ op := func() {
+ c.consumerMap.Range(func(key, value interface{}) bool {
+ consumer := value.(InnerConsumer)
+ err := consumer.PersistConsumerOffset()
+ if err != nil {
+ rlog.Error("persist offset failed", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ return true
+ })
+ }
+ time.Sleep(10 * time.Second)
+ op()
+
+ ticker := time.NewTicker(_PersistOffsetInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ op()
+ case <-c.done:
+ rlog.Info("The RMQClient stopping persist offset", map[string]interface{}{
+ "clientID": c.ClientID(),
+ })
+ return
+ }
+ }
+ })
+
+ go primitive.WithRecover(func() {
+ ticker := time.NewTicker(_RebalanceInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ticker.C:
+ c.RebalanceImmediately()
+ case <-c.done:
+ rlog.Info("The RMQClient stopping do rebalance", map[string]interface{}{
+ "clientID": c.ClientID(),
+ })
+ return
+ }
+ }
+ })
+ })
+}
+
+func (c *rmqClient) Shutdown() {
+ c.shutdownOnce.Do(func() {
+ close(c.done)
+ c.close = true
+ c.remoteClient.ShutDown()
+ })
+}
+
+func (c *rmqClient) ClientID() string {
+ id := c.option.ClientIP + "@"
+ if c.option.InstanceName == "DEFAULT" {
+ id += strconv.Itoa(os.Getpid())
+ } else {
+ id += c.option.InstanceName
+ }
+ if c.option.UnitName != "" {
+ id += "@" + c.option.UnitName
+ }
+ return id
+}
+
+func (c *rmqClient) InvokeSync(ctx context.Context, addr string, request *remote.RemotingCommand,
+ timeoutMillis time.Duration) (*remote.RemotingCommand, error) {
+ if c.close {
+ return nil, ErrServiceState
+ }
+ ctx, _ = context.WithTimeout(ctx, timeoutMillis)
+ return c.remoteClient.InvokeSync(ctx, addr, request)
+}
+
+func (c *rmqClient) InvokeAsync(ctx context.Context, addr string, request *remote.RemotingCommand,
+ f func(*remote.RemotingCommand, error)) error {
+ if c.close {
+ return ErrServiceState
+ }
+ return c.remoteClient.InvokeAsync(ctx, addr, request, func(future *remote.ResponseFuture) {
+ f(future.ResponseCommand, future.Err)
+ })
+
+}
+
+func (c *rmqClient) InvokeOneWay(ctx context.Context, addr string, request *remote.RemotingCommand,
+ timeoutMillis time.Duration) error {
+ if c.close {
+ return ErrServiceState
+ }
+ return c.remoteClient.InvokeOneWay(ctx, addr, request)
+}
+
+func (c *rmqClient) CheckClientInBroker() {
+}
+
+// TODO
+func (c *rmqClient) SendHeartbeatToAllBrokerWithLock() {
+ c.hbMutex.Lock()
+ defer c.hbMutex.Unlock()
+ hbData := NewHeartbeatData(c.ClientID())
+
+ c.producerMap.Range(func(key, value interface{}) bool {
+ pData := producerData{
+ GroupName: key.(string),
+ }
+ hbData.ProducerDatas.Add(pData)
+ return true
+ })
+
+ c.consumerMap.Range(func(key, value interface{}) bool {
+ consumer := value.(InnerConsumer)
+ cData := consumerData{
+ GroupName: key.(string),
+ CType: consumeType(consumer.GetcType()),
+ MessageModel: strings.ToUpper(consumer.GetModel()),
+ Where: consumer.GetWhere(),
+ UnitMode: consumer.IsUnitMode(),
+ SubscriptionDatas: consumer.SubscriptionDataList(),
+ }
+ hbData.ConsumerDatas.Add(cData)
+ return true
+ })
+ if hbData.ProducerDatas.Len() == 0 && hbData.ConsumerDatas.Len() == 0 {
+ rlog.Info("sending heartbeat, but no producer and no consumer", nil)
+ return
+ }
+ c.namesrvs.brokerAddressesMap.Range(func(key, value interface{}) bool {
+ brokerName := key.(string)
+ data := value.(*BrokerData)
+ for id, addr := range data.BrokerAddresses {
+ rlog.Debug("try to send heart beat to broker", map[string]interface{}{
+ "brokerName": brokerName,
+ "brokerId": id,
+ "brokerAddr": addr,
+ })
+ if hbData.ConsumerDatas.Len() == 0 && id != 0 {
+ rlog.Debug("notice, will not send heart beat to broker", map[string]interface{}{
+ "brokerName": brokerName,
+ "brokerId": id,
+ "brokerAddr": addr,
+ })
+ continue
+ }
+ cmd := remote.NewRemotingCommand(ReqHeartBeat, nil, hbData.encode())
+
+ ctx, _ := context.WithTimeout(context.Background(), 3*time.Second)
+ response, err := c.remoteClient.InvokeSync(ctx, addr, cmd)
+ if err != nil {
+ rlog.Warning("send heart beat to broker error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ return true
+ }
+ if response.Code == ResSuccess {
+ c.namesrvs.AddBrokerVersion(brokerName, addr, int32(response.Version))
+ rlog.Debug("send heart beat to broker success", map[string]interface{}{
+ "brokerName": brokerName,
+ "brokerId": id,
+ "brokerAddr": addr,
+ })
+ } else {
+ rlog.Warning("send heart beat to broker failed", map[string]interface{}{
+ "brokerName": brokerName,
+ "brokerId": id,
+ "brokerAddr": addr,
+ "responseCode": response.Code,
+ })
+ }
+ }
+ return true
+ })
+}
+
+func (c *rmqClient) UpdateTopicRouteInfo() {
+ publishTopicSet := make(map[string]bool, 0)
+ c.producerMap.Range(func(key, value interface{}) bool {
+ producer := value.(InnerProducer)
+ list := producer.PublishTopicList()
+ for idx := range list {
+ publishTopicSet[list[idx]] = true
+ }
+ return true
+ })
+ for topic := range publishTopicSet {
+ data, changed, _ := c.namesrvs.UpdateTopicRouteInfo(topic)
+ c.UpdatePublishInfo(topic, data, changed)
+ }
+
+ subscribedTopicSet := make(map[string]bool, 0)
+ c.consumerMap.Range(func(key, value interface{}) bool {
+ consumer := value.(InnerConsumer)
+ list := consumer.SubscriptionDataList()
+ for idx := range list {
+ subscribedTopicSet[list[idx].Topic] = true
+ }
+ return true
+ })
+
+ for topic := range subscribedTopicSet {
+ data, changed, _ := c.namesrvs.UpdateTopicRouteInfo(topic)
+ c.updateSubscribeInfo(topic, data, changed)
+ }
+}
+
+func (c *rmqClient) ProcessSendResponse(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) error {
+ var status primitive.SendStatus
+ switch cmd.Code {
+ case ResFlushDiskTimeout:
+ status = primitive.SendFlushDiskTimeout
+ case ResFlushSlaveTimeout:
+ status = primitive.SendFlushSlaveTimeout
+ case ResSlaveNotAvailable:
+ status = primitive.SendSlaveNotAvailable
+ case ResSuccess:
+ status = primitive.SendOK
+ default:
+ status = primitive.SendUnknownError
+ return errors.New(cmd.Remark)
+ }
+
+ msgIDs := make([]string, 0)
+ for i := 0; i < len(msgs); i++ {
+ msgIDs = append(msgIDs, msgs[i].GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex))
+ }
+ uniqueMsgId := strings.Join(msgIDs, ",")
+
+ regionId := cmd.ExtFields[primitive.PropertyMsgRegion]
+ trace := cmd.ExtFields[primitive.PropertyTraceSwitch]
+
+ if regionId == "" {
+ regionId = defaultTraceRegionID
+ }
+
+ qId, _ := strconv.Atoi(cmd.ExtFields["queueId"])
+ off, _ := strconv.ParseInt(cmd.ExtFields["queueOffset"], 10, 64)
+
+ resp.Status = status
+ resp.MsgID = uniqueMsgId
+ resp.OffsetMsgID = cmd.ExtFields["msgId"]
+ resp.MessageQueue = &primitive.MessageQueue{
+ Topic: msgs[0].Topic,
+ BrokerName: brokerName,
+ QueueId: qId,
+ }
+ resp.QueueOffset = off
+ resp.TransactionID = cmd.ExtFields["transactionId"]
+ resp.RegionID = regionId
+ resp.TraceOn = trace != "" && trace != _TranceOff
+ return nil
+}
+
+// PullMessage with sync
+func (c *rmqClient) PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequestHeader) (*primitive.PullResult, error) {
+ cmd := remote.NewRemotingCommand(ReqPullMessage, request, nil)
+ ctx, _ = context.WithTimeout(ctx, 30*time.Second)
+ res, err := c.remoteClient.InvokeSync(ctx, brokerAddrs, cmd)
+ if err != nil {
+ return nil, err
+ }
+
+ return c.processPullResponse(res)
+}
+
+func (c *rmqClient) processPullResponse(response *remote.RemotingCommand) (*primitive.PullResult, error) {
+
+ pullResult := &primitive.PullResult{}
+ switch response.Code {
+ case ResSuccess:
+ pullResult.Status = primitive.PullFound
+ case ResPullNotFound:
+ pullResult.Status = primitive.PullNoNewMsg
+ case ResPullRetryImmediately:
+ pullResult.Status = primitive.PullNoMsgMatched
+ case ResPullOffsetMoved:
+ pullResult.Status = primitive.PullOffsetIllegal
+ default:
+ return nil, fmt.Errorf("unknown Response Code: %d, remark: %s", response.Code, response.Remark)
+ }
+
+ c.decodeCommandCustomHeader(pullResult, response)
+ pullResult.SetBody(response.Body)
+
+ return pullResult, nil
+}
+
+func (c *rmqClient) decodeCommandCustomHeader(pr *primitive.PullResult, cmd *remote.RemotingCommand) {
+ v, exist := cmd.ExtFields["maxOffset"]
+ if exist {
+ pr.MaxOffset, _ = strconv.ParseInt(v, 10, 64)
+ }
+
+ v, exist = cmd.ExtFields["minOffset"]
+ if exist {
+ pr.MinOffset, _ = strconv.ParseInt(v, 10, 64)
+ }
+
+ v, exist = cmd.ExtFields["nextBeginOffset"]
+ if exist {
+ pr.NextBeginOffset, _ = strconv.ParseInt(v, 10, 64)
+ }
+
+ v, exist = cmd.ExtFields["suggestWhichBrokerId"]
+ if exist {
+ pr.SuggestWhichBrokerId, _ = strconv.ParseInt(v, 10, 64)
+ }
+}
+
+func (c *rmqClient) RegisterConsumer(group string, consumer InnerConsumer) error {
+ _, exist := c.consumerMap.Load(group)
+ if exist {
+ rlog.Warning("the consumer group exist already", map[string]interface{}{
+ rlog.LogKeyConsumerGroup: group,
+ })
+ return fmt.Errorf("the consumer group exist already")
+ }
+ c.consumerMap.Store(group, consumer)
+ return nil
+}
+
+func (c *rmqClient) UnregisterConsumer(group string) {
+ c.consumerMap.Delete(group)
+}
+
+func (c *rmqClient) RegisterProducer(group string, producer InnerProducer) {
+ c.producerMap.Store(group, producer)
+}
+
+func (c *rmqClient) UnregisterProducer(group string) {
+ c.producerMap.Delete(group)
+}
+
+func (c *rmqClient) RebalanceImmediately() {
+ c.rbMutex.Lock()
+ defer c.rbMutex.Unlock()
+ c.consumerMap.Range(func(key, value interface{}) bool {
+ consumer := value.(InnerConsumer)
+ consumer.Rebalance()
+ return true
+ })
+}
+
+func (c *rmqClient) UpdatePublishInfo(topic string, data *TopicRouteData, changed bool) {
+ if data == nil {
+ return
+ }
+
+ c.producerMap.Range(func(key, value interface{}) bool {
+ p := value.(InnerProducer)
+ updated := changed
+ if !updated {
+ updated = p.IsPublishTopicNeedUpdate(topic)
+ }
+ if updated {
+ publishInfo := c.namesrvs.routeData2PublishInfo(topic, data)
+ publishInfo.HaveTopicRouterInfo = true
+ p.UpdateTopicPublishInfo(topic, publishInfo)
+ }
+ return true
+ })
+}
+
+func (c *rmqClient) updateSubscribeInfo(topic string, data *TopicRouteData, changed bool) {
+ if data == nil {
+ return
+ }
+ c.consumerMap.Range(func(key, value interface{}) bool {
+ consumer := value.(InnerConsumer)
+ updated := changed
+ if !updated {
+ updated = consumer.IsSubscribeTopicNeedUpdate(topic)
+ }
+ if updated {
+ consumer.UpdateTopicSubscribeInfo(topic, routeData2SubscribeInfo(topic, data))
+ }
+
+ return true
+ })
+}
+
+func (c *rmqClient) isNeedUpdateSubscribeInfo(topic string) bool {
+ var result bool
+ c.consumerMap.Range(func(key, value interface{}) bool {
+ consumer := value.(InnerConsumer)
+ if consumer.IsSubscribeTopicNeedUpdate(topic) {
+ result = true
+ return false
+ }
+ return true
+ })
+ return result
+}
+
+func (c *rmqClient) getConsumerRunningInfo(group string) *ConsumerRunningInfo {
+ consumer, exist := c.consumerMap.Load(group)
+ if !exist {
+ return nil
+ }
+ info := consumer.(InnerConsumer).GetConsumerRunningInfo()
+ if info != nil {
+ info.Properties[PropClientVersion] = clientVersion
+ }
+ return info
+}
+
+func routeData2SubscribeInfo(topic string, data *TopicRouteData) []*primitive.MessageQueue {
+ list := make([]*primitive.MessageQueue, 0)
+ for idx := range data.QueueDataList {
+ qd := data.QueueDataList[idx]
+ if queueIsReadable(qd.Perm) {
+ for i := 0; i < qd.ReadQueueNums; i++ {
+ list = append(list, &primitive.MessageQueue{
+ Topic: topic,
+ BrokerName: qd.BrokerName,
+ QueueId: i,
+ })
+ }
+ }
+ }
+ return list
+}
+
+func brokerVIPChannel(brokerAddr string) string {
+ if !_VIPChannelEnable {
+ return brokerAddr
+ }
+ var brokerAddrNew strings.Builder
+ ipAndPort := strings.Split(brokerAddr, ":")
+ port, err := strconv.Atoi(ipAndPort[1])
+ if err != nil {
+ return ""
+ }
+ brokerAddrNew.WriteString(ipAndPort[0])
+ brokerAddrNew.WriteString(":")
+ brokerAddrNew.WriteString(strconv.Itoa(port - 2))
+ return brokerAddrNew.String()
+}
diff --git a/internal/constants.go b/internal/constants.go
new file mode 100644
index 0000000..e2e911f
--- /dev/null
+++ b/internal/constants.go
@@ -0,0 +1,29 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+const (
+ RetryGroupTopicPrefix = "%RETRY%"
+ DefaultConsumerGroup = "DEFAULT_CONSUMER"
+ ClientInnerProducerGroup = "CLIENT_INNER_PRODUCER"
+ SystemTopicPrefix = "rmq_sys_"
+)
+
+func GetRetryTopic(group string) string {
+ return RetryGroupTopicPrefix + group
+}
diff --git a/internal/mock_client.go b/internal/mock_client.go
new file mode 100644
index 0000000..ab34ac1
--- /dev/null
+++ b/internal/mock_client.go
@@ -0,0 +1,419 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by MockGen. DO NOT EDIT.
+// Source: client.go
+
+// Package internal is a generated GoMock package.
+package internal
+
+import (
+ "context"
+ "reflect"
+ "time"
+
+ "github.com/golang/mock/gomock"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+// MockInnerProducer is a mock of InnerProducer interface
+type MockInnerProducer struct {
+ ctrl *gomock.Controller
+ recorder *MockInnerProducerMockRecorder
+}
+
+// MockInnerProducerMockRecorder is the mock recorder for MockInnerProducer
+type MockInnerProducerMockRecorder struct {
+ mock *MockInnerProducer
+}
+
+// NewMockInnerProducer creates a new mock instance
+func NewMockInnerProducer(ctrl *gomock.Controller) *MockInnerProducer {
+ mock := &MockInnerProducer{ctrl: ctrl}
+ mock.recorder = &MockInnerProducerMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockInnerProducer) EXPECT() *MockInnerProducerMockRecorder {
+ return m.recorder
+}
+
+// PublishTopicList mocks base method
+func (m *MockInnerProducer) PublishTopicList() []string {
+ ret := m.ctrl.Call(m, "PublishTopicList")
+ ret0, _ := ret[0].([]string)
+ return ret0
+}
+
+// PublishTopicList indicates an expected call of PublishTopicList
+func (mr *MockInnerProducerMockRecorder) PublishTopicList() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PublishTopicList", reflect.TypeOf((*MockInnerProducer)(nil).PublishTopicList))
+}
+
+// UpdateTopicPublishInfo mocks base method
+func (m *MockInnerProducer) UpdateTopicPublishInfo(topic string, info *TopicPublishInfo) {
+ m.ctrl.Call(m, "UpdateTopicPublishInfo", topic, info)
+}
+
+// UpdateTopicPublishInfo indicates an expected call of UpdateTopicPublishInfo
+func (mr *MockInnerProducerMockRecorder) UpdateTopicPublishInfo(topic, info interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicPublishInfo", reflect.TypeOf((*MockInnerProducer)(nil).UpdateTopicPublishInfo), topic, info)
+}
+
+// IsPublishTopicNeedUpdate mocks base method
+func (m *MockInnerProducer) IsPublishTopicNeedUpdate(topic string) bool {
+ ret := m.ctrl.Call(m, "IsPublishTopicNeedUpdate", topic)
+ ret0, _ := ret[0].(bool)
+ return ret0
+}
+
+// IsPublishTopicNeedUpdate indicates an expected call of IsPublishTopicNeedUpdate
+func (mr *MockInnerProducerMockRecorder) IsPublishTopicNeedUpdate(topic interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPublishTopicNeedUpdate", reflect.TypeOf((*MockInnerProducer)(nil).IsPublishTopicNeedUpdate), topic)
+}
+
+// IsUnitMode mocks base method
+func (m *MockInnerProducer) IsUnitMode() bool {
+ ret := m.ctrl.Call(m, "IsUnitMode")
+ ret0, _ := ret[0].(bool)
+ return ret0
+}
+
+// IsUnitMode indicates an expected call of IsUnitMode
+func (mr *MockInnerProducerMockRecorder) IsUnitMode() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnitMode", reflect.TypeOf((*MockInnerProducer)(nil).IsUnitMode))
+}
+
+// MockInnerConsumer is a mock of InnerConsumer interface
+type MockInnerConsumer struct {
+ ctrl *gomock.Controller
+ recorder *MockInnerConsumerMockRecorder
+}
+
+// MockInnerConsumerMockRecorder is the mock recorder for MockInnerConsumer
+type MockInnerConsumerMockRecorder struct {
+ mock *MockInnerConsumer
+}
+
+// NewMockInnerConsumer creates a new mock instance
+func NewMockInnerConsumer(ctrl *gomock.Controller) *MockInnerConsumer {
+ mock := &MockInnerConsumer{ctrl: ctrl}
+ mock.recorder = &MockInnerConsumerMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockInnerConsumer) EXPECT() *MockInnerConsumerMockRecorder {
+ return m.recorder
+}
+
+// PersistConsumerOffset mocks base method
+func (m *MockInnerConsumer) PersistConsumerOffset() error {
+ ret := m.ctrl.Call(m, "PersistConsumerOffset")
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// PersistConsumerOffset indicates an expected call of PersistConsumerOffset
+func (mr *MockInnerConsumerMockRecorder) PersistConsumerOffset() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PersistConsumerOffset", reflect.TypeOf((*MockInnerConsumer)(nil).PersistConsumerOffset))
+}
+
+// UpdateTopicSubscribeInfo mocks base method
+func (m *MockInnerConsumer) UpdateTopicSubscribeInfo(topic string, mqs []*primitive.MessageQueue) {
+ m.ctrl.Call(m, "UpdateTopicSubscribeInfo", topic, mqs)
+}
+
+// UpdateTopicSubscribeInfo indicates an expected call of UpdateTopicSubscribeInfo
+func (mr *MockInnerConsumerMockRecorder) UpdateTopicSubscribeInfo(topic, mqs interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicSubscribeInfo", reflect.TypeOf((*MockInnerConsumer)(nil).UpdateTopicSubscribeInfo), topic, mqs)
+}
+
+// IsSubscribeTopicNeedUpdate mocks base method
+func (m *MockInnerConsumer) IsSubscribeTopicNeedUpdate(topic string) bool {
+ ret := m.ctrl.Call(m, "IsSubscribeTopicNeedUpdate", topic)
+ ret0, _ := ret[0].(bool)
+ return ret0
+}
+
+// IsSubscribeTopicNeedUpdate indicates an expected call of IsSubscribeTopicNeedUpdate
+func (mr *MockInnerConsumerMockRecorder) IsSubscribeTopicNeedUpdate(topic interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsSubscribeTopicNeedUpdate", reflect.TypeOf((*MockInnerConsumer)(nil).IsSubscribeTopicNeedUpdate), topic)
+}
+
+// SubscriptionDataList mocks base method
+func (m *MockInnerConsumer) SubscriptionDataList() []*SubscriptionData {
+ ret := m.ctrl.Call(m, "SubscriptionDataList")
+ ret0, _ := ret[0].([]*SubscriptionData)
+ return ret0
+}
+
+// SubscriptionDataList indicates an expected call of SubscriptionDataList
+func (mr *MockInnerConsumerMockRecorder) SubscriptionDataList() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubscriptionDataList", reflect.TypeOf((*MockInnerConsumer)(nil).SubscriptionDataList))
+}
+
+// Rebalance mocks base method
+func (m *MockInnerConsumer) Rebalance() {
+ m.ctrl.Call(m, "Rebalance")
+}
+
+// Rebalance indicates an expected call of Rebalance
+func (mr *MockInnerConsumerMockRecorder) Rebalance() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rebalance", reflect.TypeOf((*MockInnerConsumer)(nil).Rebalance))
+}
+
+// IsUnitMode mocks base method
+func (m *MockInnerConsumer) IsUnitMode() bool {
+ ret := m.ctrl.Call(m, "IsUnitMode")
+ ret0, _ := ret[0].(bool)
+ return ret0
+}
+
+// IsUnitMode indicates an expected call of IsUnitMode
+func (mr *MockInnerConsumerMockRecorder) IsUnitMode() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUnitMode", reflect.TypeOf((*MockInnerConsumer)(nil).IsUnitMode))
+}
+
+// GetConsumerRunningInfo mocks base method
+func (m *MockInnerConsumer) GetConsumerRunningInfo() *ConsumerRunningInfo {
+ ret := m.ctrl.Call(m, "GetConsumerRunningInfo")
+ ret0, _ := ret[0].(*ConsumerRunningInfo)
+ return ret0
+}
+
+// GetConsumerRunningInfo indicates an expected call of GetConsumerRunningInfo
+func (mr *MockInnerConsumerMockRecorder) GetConsumerRunningInfo() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConsumerRunningInfo", reflect.TypeOf((*MockInnerConsumer)(nil).GetConsumerRunningInfo))
+}
+
+// MockRMQClient is a mock of RMQClient interface
+type MockRMQClient struct {
+ ctrl *gomock.Controller
+ recorder *MockRMQClientMockRecorder
+}
+
+// MockRMQClientMockRecorder is the mock recorder for MockRMQClient
+type MockRMQClientMockRecorder struct {
+ mock *MockRMQClient
+}
+
+// NewMockRMQClient creates a new mock instance
+func NewMockRMQClient(ctrl *gomock.Controller) *MockRMQClient {
+ mock := &MockRMQClient{ctrl: ctrl}
+ mock.recorder = &MockRMQClientMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockRMQClient) EXPECT() *MockRMQClientMockRecorder {
+ return m.recorder
+}
+
+// Start mocks base method
+func (m *MockRMQClient) Start() {
+ m.ctrl.Call(m, "Start")
+}
+
+// Start indicates an expected call of Start
+func (mr *MockRMQClientMockRecorder) Start() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockRMQClient)(nil).Start))
+}
+
+// Shutdown mocks base method
+func (m *MockRMQClient) Shutdown() {
+ m.ctrl.Call(m, "Shutdown")
+}
+
+// Shutdown indicates an expected call of Shutdown
+func (mr *MockRMQClientMockRecorder) Shutdown() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockRMQClient)(nil).Shutdown))
+}
+
+// ClientID mocks base method
+func (m *MockRMQClient) ClientID() string {
+ ret := m.ctrl.Call(m, "ClientID")
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// ClientID indicates an expected call of ClientID
+func (mr *MockRMQClientMockRecorder) ClientID() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ClientID", reflect.TypeOf((*MockRMQClient)(nil).ClientID))
+}
+
+// RegisterProducer mocks base method
+func (m *MockRMQClient) RegisterProducer(group string, producer InnerProducer) {
+ m.ctrl.Call(m, "RegisterProducer", group, producer)
+}
+
+// RegisterProducer indicates an expected call of RegisterProducer
+func (mr *MockRMQClientMockRecorder) RegisterProducer(group, producer interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterProducer", reflect.TypeOf((*MockRMQClient)(nil).RegisterProducer), group, producer)
+}
+
+// UnregisterProducer mocks base method
+func (m *MockRMQClient) UnregisterProducer(group string) {
+ m.ctrl.Call(m, "UnregisterProducer", group)
+}
+
+// UnregisterProducer indicates an expected call of UnregisterProducer
+func (mr *MockRMQClientMockRecorder) UnregisterProducer(group interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnregisterProducer", reflect.TypeOf((*MockRMQClient)(nil).UnregisterProducer), group)
+}
+
+// InvokeSync mocks base method
+func (m *MockRMQClient) InvokeSync(ctx context.Context, addr string, request *remote.RemotingCommand, timeoutMillis time.Duration) (*remote.RemotingCommand, error) {
+ ret := m.ctrl.Call(m, "InvokeSync", ctx, addr, request, timeoutMillis)
+ ret0, _ := ret[0].(*remote.RemotingCommand)
+ ret1, _ := ret[1].(error)
+ return ret0, ret1
+}
+
+// InvokeSync indicates an expected call of InvokeSync
+func (mr *MockRMQClientMockRecorder) InvokeSync(ctx, addr, request, timeoutMillis interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeSync", reflect.TypeOf((*MockRMQClient)(nil).InvokeSync), ctx, addr, request, timeoutMillis)
+}
+
+// InvokeAsync mocks base method
+func (m *MockRMQClient) InvokeAsync(ctx context.Context, addr string, request *remote.RemotingCommand, f func(*remote.RemotingCommand, error)) error {
+ ret := m.ctrl.Call(m, "InvokeAsync", ctx, addr, request, f)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// InvokeAsync indicates an expected call of InvokeAsync
+func (mr *MockRMQClientMockRecorder) InvokeAsync(ctx, addr, request, f interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeAsync", reflect.TypeOf((*MockRMQClient)(nil).InvokeAsync), ctx, addr, request, f)
+}
+
+// InvokeOneWay mocks base method
+func (m *MockRMQClient) InvokeOneWay(ctx context.Context, addr string, request *remote.RemotingCommand, timeoutMillis time.Duration) error {
+ ret := m.ctrl.Call(m, "InvokeOneWay", ctx, addr, request, timeoutMillis)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// InvokeOneWay indicates an expected call of InvokeOneWay
+func (mr *MockRMQClientMockRecorder) InvokeOneWay(ctx, addr, request, timeoutMillis interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeOneWay", reflect.TypeOf((*MockRMQClient)(nil).InvokeOneWay), ctx, addr, request, timeoutMillis)
+}
+
+// CheckClientInBroker mocks base method
+func (m *MockRMQClient) CheckClientInBroker() {
+ m.ctrl.Call(m, "CheckClientInBroker")
+}
+
+// CheckClientInBroker indicates an expected call of CheckClientInBroker
+func (mr *MockRMQClientMockRecorder) CheckClientInBroker() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckClientInBroker", reflect.TypeOf((*MockRMQClient)(nil).CheckClientInBroker))
+}
+
+// SendHeartbeatToAllBrokerWithLock mocks base method
+func (m *MockRMQClient) SendHeartbeatToAllBrokerWithLock() {
+ m.ctrl.Call(m, "SendHeartbeatToAllBrokerWithLock")
+}
+
+// SendHeartbeatToAllBrokerWithLock indicates an expected call of SendHeartbeatToAllBrokerWithLock
+func (mr *MockRMQClientMockRecorder) SendHeartbeatToAllBrokerWithLock() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendHeartbeatToAllBrokerWithLock", reflect.TypeOf((*MockRMQClient)(nil).SendHeartbeatToAllBrokerWithLock))
+}
+
+// UpdateTopicRouteInfo mocks base method
+func (m *MockRMQClient) UpdateTopicRouteInfo() {
+ m.ctrl.Call(m, "UpdateTopicRouteInfo")
+}
+
+// UpdateTopicRouteInfo indicates an expected call of UpdateTopicRouteInfo
+func (mr *MockRMQClientMockRecorder) UpdateTopicRouteInfo() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicRouteInfo", reflect.TypeOf((*MockRMQClient)(nil).UpdateTopicRouteInfo))
+}
+
+// ProcessSendResponse mocks base method
+func (m *MockRMQClient) ProcessSendResponse(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) error {
+ varargs := []interface{}{brokerName, cmd, resp}
+ for _, a := range msgs {
+ varargs = append(varargs, a)
+ }
+ ret := m.ctrl.Call(m, "ProcessSendResponse", varargs...)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// ProcessSendResponse indicates an expected call of ProcessSendResponse
+func (mr *MockRMQClientMockRecorder) ProcessSendResponse(brokerName, cmd, resp interface{}, msgs ...interface{}) *gomock.Call {
+ varargs := append([]interface{}{brokerName, cmd, resp}, msgs...)
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessSendResponse", reflect.TypeOf((*MockRMQClient)(nil).ProcessSendResponse), varargs...)
+}
+
+// RegisterConsumer mocks base method
+func (m *MockRMQClient) RegisterConsumer(group string, consumer InnerConsumer) error {
+ ret := m.ctrl.Call(m, "RegisterConsumer", group, consumer)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// RegisterConsumer indicates an expected call of RegisterConsumer
+func (mr *MockRMQClientMockRecorder) RegisterConsumer(group, consumer interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterConsumer", reflect.TypeOf((*MockRMQClient)(nil).RegisterConsumer), group, consumer)
+}
+
+// UnregisterConsumer mocks base method
+func (m *MockRMQClient) UnregisterConsumer(group string) {
+ m.ctrl.Call(m, "UnregisterConsumer", group)
+}
+
+// UnregisterConsumer indicates an expected call of UnregisterConsumer
+func (mr *MockRMQClientMockRecorder) UnregisterConsumer(group interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnregisterConsumer", reflect.TypeOf((*MockRMQClient)(nil).UnregisterConsumer), group)
+}
+
+// PullMessage mocks base method
+func (m *MockRMQClient) PullMessage(ctx context.Context, brokerAddrs string, request *PullMessageRequestHeader) (*primitive.PullResult, error) {
+ ret := m.ctrl.Call(m, "PullMessage", ctx, brokerAddrs, request)
+ ret0, _ := ret[0].(*primitive.PullResult)
+ ret1, _ := ret[1].(error)
+ return ret0, ret1
+}
+
+// PullMessage indicates an expected call of PullMessage
+func (mr *MockRMQClientMockRecorder) PullMessage(ctx, brokerAddrs, request interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PullMessage", reflect.TypeOf((*MockRMQClient)(nil).PullMessage), ctx, brokerAddrs, request)
+}
+
+// RebalanceImmediately mocks base method
+func (m *MockRMQClient) RebalanceImmediately() {
+ m.ctrl.Call(m, "RebalanceImmediately")
+}
+
+// RebalanceImmediately indicates an expected call of RebalanceImmediately
+func (mr *MockRMQClientMockRecorder) RebalanceImmediately() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebalanceImmediately", reflect.TypeOf((*MockRMQClient)(nil).RebalanceImmediately))
+}
+
+// UpdatePublishInfo mocks base method
+func (m *MockRMQClient) UpdatePublishInfo(topic string, data *TopicRouteData, changed bool) {
+ m.ctrl.Call(m, "UpdatePublishInfo", topic, data, changed)
+}
+
+// UpdatePublishInfo indicates an expected call of UpdatePublishInfo
+func (mr *MockRMQClientMockRecorder) UpdatePublishInfo(topic, data, changed interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePublishInfo", reflect.TypeOf((*MockRMQClient)(nil).UpdatePublishInfo), topic, data, changed)
+}
diff --git a/internal/mock_namesrv.go b/internal/mock_namesrv.go
new file mode 100644
index 0000000..f87d174
--- /dev/null
+++ b/internal/mock_namesrv.go
@@ -0,0 +1,189 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+// Code generated by MockGen. DO NOT EDIT.
+// Source: namesrv.go
+
+// Package internal is a generated GoMock package.
+package internal
+
+import (
+ reflect "reflect"
+
+ primitive "github.com/apache/rocketmq-client-go/v2/primitive"
+ gomock "github.com/golang/mock/gomock"
+)
+
+// MockNamesrvs is a mock of Namesrvs interface
+type MockNamesrvs struct {
+ ctrl *gomock.Controller
+ recorder *MockNamesrvsMockRecorder
+}
+
+// MockNamesrvsMockRecorder is the mock recorder for MockNamesrvs
+type MockNamesrvsMockRecorder struct {
+ mock *MockNamesrvs
+}
+
+// NewMockNamesrvs creates a new mock instance
+func NewMockNamesrvs(ctrl *gomock.Controller) *MockNamesrvs {
+ mock := &MockNamesrvs{ctrl: ctrl}
+ mock.recorder = &MockNamesrvsMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockNamesrvs) EXPECT() *MockNamesrvsMockRecorder {
+ return m.recorder
+}
+
+// UpdateNameServerAddress mocks base method
+func (m *MockNamesrvs) UpdateNameServerAddress() {
+ m.ctrl.T.Helper()
+ m.ctrl.Call(m, "UpdateNameServerAddress")
+}
+
+// UpdateNameServerAddress indicates an expected call of UpdateNameServerAddress
+func (mr *MockNamesrvsMockRecorder) UpdateNameServerAddress() *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateNameServerAddress", reflect.TypeOf((*MockNamesrvs)(nil).UpdateNameServerAddress))
+}
+
+// AddBroker mocks base method
+func (m *MockNamesrvs) AddBroker(routeData *TopicRouteData) {
+ m.ctrl.T.Helper()
+ m.ctrl.Call(m, "AddBroker", routeData)
+}
+
+// AddBroker indicates an expected call of AddBroker
+func (mr *MockNamesrvsMockRecorder) AddBroker(routeData interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddBroker", reflect.TypeOf((*MockNamesrvs)(nil).AddBroker), routeData)
+}
+
+// cleanOfflineBroker mocks base method
+func (m *MockNamesrvs) cleanOfflineBroker() {
+ m.ctrl.T.Helper()
+ m.ctrl.Call(m, "cleanOfflineBroker")
+}
+
+// cleanOfflineBroker indicates an expected call of cleanOfflineBroker
+func (mr *MockNamesrvsMockRecorder) cleanOfflineBroker() *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "cleanOfflineBroker", reflect.TypeOf((*MockNamesrvs)(nil).cleanOfflineBroker))
+}
+
+// UpdateTopicRouteInfo mocks base method
+func (m *MockNamesrvs) UpdateTopicRouteInfo(topic string) (*TopicRouteData, bool, error) {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "UpdateTopicRouteInfo", topic)
+ ret0, _ := ret[0].(*TopicRouteData)
+ ret1, _ := ret[1].(bool)
+ ret2, _ := ret[2].(error)
+ return ret0, ret1, ret2
+}
+
+// UpdateTopicRouteInfo indicates an expected call of UpdateTopicRouteInfo
+func (mr *MockNamesrvsMockRecorder) UpdateTopicRouteInfo(topic interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateTopicRouteInfo", reflect.TypeOf((*MockNamesrvs)(nil).UpdateTopicRouteInfo), topic)
+}
+
+// FetchPublishMessageQueues mocks base method
+func (m *MockNamesrvs) FetchPublishMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "FetchPublishMessageQueues", topic)
+ ret0, _ := ret[0].([]*primitive.MessageQueue)
+ ret1, _ := ret[1].(error)
+ return ret0, ret1
+}
+
+// FetchPublishMessageQueues indicates an expected call of FetchPublishMessageQueues
+func (mr *MockNamesrvsMockRecorder) FetchPublishMessageQueues(topic interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchPublishMessageQueues", reflect.TypeOf((*MockNamesrvs)(nil).FetchPublishMessageQueues), topic)
+}
+
+// FindBrokerAddrByTopic mocks base method
+func (m *MockNamesrvs) FindBrokerAddrByTopic(topic string) string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "FindBrokerAddrByTopic", topic)
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// FindBrokerAddrByTopic indicates an expected call of FindBrokerAddrByTopic
+func (mr *MockNamesrvsMockRecorder) FindBrokerAddrByTopic(topic interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBrokerAddrByTopic", reflect.TypeOf((*MockNamesrvs)(nil).FindBrokerAddrByTopic), topic)
+}
+
+// FindBrokerAddrByName mocks base method
+func (m *MockNamesrvs) FindBrokerAddrByName(brokerName string) string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "FindBrokerAddrByName", brokerName)
+ ret0, _ := ret[0].(string)
+ return ret0
+}
+
+// FindBrokerAddrByName indicates an expected call of FindBrokerAddrByName
+func (mr *MockNamesrvsMockRecorder) FindBrokerAddrByName(brokerName interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBrokerAddrByName", reflect.TypeOf((*MockNamesrvs)(nil).FindBrokerAddrByName), brokerName)
+}
+
+// FindBrokerAddressInSubscribe mocks base method
+func (m *MockNamesrvs) FindBrokerAddressInSubscribe(brokerName string, brokerId int64, onlyThisBroker bool) *FindBrokerResult {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "FindBrokerAddressInSubscribe", brokerName, brokerId, onlyThisBroker)
+ ret0, _ := ret[0].(*FindBrokerResult)
+ return ret0
+}
+
+// FindBrokerAddressInSubscribe indicates an expected call of FindBrokerAddressInSubscribe
+func (mr *MockNamesrvsMockRecorder) FindBrokerAddressInSubscribe(brokerName, brokerId, onlyThisBroker interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindBrokerAddressInSubscribe", reflect.TypeOf((*MockNamesrvs)(nil).FindBrokerAddressInSubscribe), brokerName, brokerId, onlyThisBroker)
+}
+
+// FetchSubscribeMessageQueues mocks base method
+func (m *MockNamesrvs) FetchSubscribeMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "FetchSubscribeMessageQueues", topic)
+ ret0, _ := ret[0].([]*primitive.MessageQueue)
+ ret1, _ := ret[1].(error)
+ return ret0, ret1
+}
+
+// FetchSubscribeMessageQueues indicates an expected call of FetchSubscribeMessageQueues
+func (mr *MockNamesrvsMockRecorder) FetchSubscribeMessageQueues(topic interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FetchSubscribeMessageQueues", reflect.TypeOf((*MockNamesrvs)(nil).FetchSubscribeMessageQueues), topic)
+}
+
+// AddrList mocks base method
+func (m *MockNamesrvs) AddrList() []string {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "AddrList")
+ ret0, _ := ret[0].([]string)
+ return ret0
+}
+
+// AddrList indicates an expected call of AddrList
+func (mr *MockNamesrvsMockRecorder) AddrList() *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddrList", reflect.TypeOf((*MockNamesrvs)(nil).AddrList))
+}
diff --git a/internal/model.go b/internal/model.go
new file mode 100644
index 0000000..0ee9ccc
--- /dev/null
+++ b/internal/model.go
@@ -0,0 +1,263 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "sort"
+ "strings"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+ jsoniter "github.com/json-iterator/go"
+)
+
+type FindBrokerResult struct {
+ BrokerAddr string
+ Slave bool
+ BrokerVersion int32
+}
+
+type (
+ // groupName of consumer
+ consumeType string
+
+ ServiceState int32
+)
+
+const (
+ StateCreateJust ServiceState = iota
+ StateStartFailed
+ StateRunning
+ StateShutdown
+)
+
+type SubscriptionData struct {
+ ClassFilterMode bool `json:"classFilterMode"`
+ Topic string `json:"topic"`
+ SubString string `json:"subString"`
+ Tags utils.Set `json:"tagsSet"`
+ Codes utils.Set `json:"codeSet"`
+ SubVersion int64 `json:"subVersion"`
+ ExpType string `json:"expressionType"`
+}
+
+type producerData struct {
+ GroupName string `json:"groupName"`
+}
+
+func (p producerData) UniqueID() string {
+ return p.GroupName
+}
+
+type consumerData struct {
+ GroupName string `json:"groupName"`
+ CType consumeType `json:"consumeType"`
+ MessageModel string `json:"messageModel"`
+ Where string `json:"consumeFromWhere"`
+ SubscriptionDatas []*SubscriptionData `json:"subscriptionDataSet"`
+ UnitMode bool `json:"unitMode"`
+}
+
+func (c consumerData) UniqueID() string {
+ return c.GroupName
+}
+
+type heartbeatData struct {
+ ClientId string `json:"clientID"`
+ ProducerDatas utils.Set `json:"producerDataSet"`
+ ConsumerDatas utils.Set `json:"consumerDataSet"`
+}
+
+func NewHeartbeatData(clientID string) *heartbeatData {
+ return &heartbeatData{
+ ClientId: clientID,
+ ProducerDatas: utils.NewSet(),
+ ConsumerDatas: utils.NewSet(),
+ }
+}
+
+func (data *heartbeatData) encode() []byte {
+ d, err := jsoniter.Marshal(data)
+ if err != nil {
+ rlog.Error("marshal heartbeatData error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ return nil
+ }
+ rlog.Debug("heartbeat: "+string(d), nil)
+ return d
+}
+
+const (
+ PropNameServerAddr = "PROP_NAMESERVER_ADDR"
+ PropThreadPoolCoreSize = "PROP_THREADPOOL_CORE_SIZE"
+ PropConsumeOrderly = "PROP_CONSUMEORDERLY"
+ PropConsumeType = "PROP_CONSUME_TYPE"
+ PropClientVersion = "PROP_CLIENT_VERSION"
+ PropConsumerStartTimestamp = "PROP_CONSUMER_START_TIMESTAMP"
+)
+
+type ProcessQueueInfo struct {
+ CommitOffset int64 `json:"commitOffset"`
+ CachedMsgMinOffset int64 `json:"cachedMsgMinOffset"`
+ CachedMsgMaxOffset int64 `json:"cachedMsgMaxOffset"`
+ CachedMsgCount int `json:"cachedMsgCount"`
+ CachedMsgSizeInMiB int64 `json:"cachedMsgSizeInMiB"`
+ TransactionMsgMinOffset int64 `json:"transactionMsgMinOffset"`
+ TransactionMsgMaxOffset int64 `json:"transactionMsgMaxOffset"`
+ TransactionMsgCount int `json:"transactionMsgCount"`
+ Locked bool `json:"locked"`
+ TryUnlockTimes int64 `json:"tryUnlockTimes"`
+ LastLockTimestamp int64 `json:"lastLockTimestamp"`
+ Dropped bool `json:"dropped"`
+ LastPullTimestamp int64 `json:"lastPullTimestamp"`
+ LastConsumeTimestamp int64 `json:"lastConsumeTimestamp"`
+}
+
+type ConsumeStatus struct {
+ PullRT float64 `json:"pullRT"`
+ PullTPS float64 `json:"pullTPS"`
+ ConsumeRT float64 `json:"consumeRT"`
+ ConsumeOKTPS float64 `json:"consumeOKTPS"`
+ ConsumeFailedTPS float64 `json:"consumeFailedTPS"`
+ ConsumeFailedMsgs int64 `json:"consumeFailedMsgs"`
+}
+
+type ConsumerRunningInfo struct {
+ Properties map[string]string
+ SubscriptionData map[*SubscriptionData]bool
+ MQTable map[primitive.MessageQueue]ProcessQueueInfo
+ StatusTable map[string]ConsumeStatus
+}
+
+func (info ConsumerRunningInfo) Encode() ([]byte, error) {
+ data, err := json.Marshal(info.Properties)
+ if err != nil {
+ return nil, err
+ }
+ jsonData := fmt.Sprintf("{\"%s\":%s", "properties", string(data))
+
+ data, err = json.Marshal(info.StatusTable)
+ if err != nil {
+ return nil, err
+ }
+ jsonData = fmt.Sprintf("%s,\"%s\":%s", jsonData, "statusTable", string(data))
+
+ subs := make([]*SubscriptionData, len(info.SubscriptionData))
+ idx := 0
+ for k := range info.SubscriptionData {
+ subs[idx] = k
+ idx++
+ }
+
+ // make sure test case table
+ sort.Slice(subs, func(i, j int) bool {
+ sub1 := subs[i]
+ sub2 := subs[j]
+ if sub1.ClassFilterMode != sub2.ClassFilterMode {
+ return sub1.ClassFilterMode == false
+ }
+ com := strings.Compare(sub1.Topic, sub1.Topic)
+ if com != 0 {
+ return com > 0
+ }
+
+ com = strings.Compare(sub1.SubString, sub1.SubString)
+ if com != 0 {
+ return com > 0
+ }
+
+ if sub1.SubVersion != sub2.SubVersion {
+ return sub1.SubVersion > sub2.SubVersion
+ }
+
+ com = strings.Compare(sub1.ExpType, sub1.ExpType)
+ if com != 0 {
+ return com > 0
+ }
+
+ v1, _ := sub1.Tags.MarshalJSON()
+ v2, _ := sub2.Tags.MarshalJSON()
+ com = bytes.Compare(v1, v2)
+ if com != 0 {
+ return com > 0
+ }
+
+ v1, _ = sub1.Codes.MarshalJSON()
+ v2, _ = sub2.Codes.MarshalJSON()
+ com = bytes.Compare(v1, v2)
+ if com != 0 {
+ return com > 0
+ }
+ return true
+ })
+
+ data, err = json.Marshal(subs)
+ if err != nil {
+ return nil, err
+ }
+ jsonData = fmt.Sprintf("%s,\"%s\":%s", jsonData, "subscriptionSet", string(data))
+
+ tableJson := ""
+ keys := make([]primitive.MessageQueue, 0)
+
+ for k := range info.MQTable {
+ keys = append(keys, k)
+ }
+
+ sort.Slice(keys, func(i, j int) bool {
+ q1 := keys[i]
+ q2 := keys[j]
+ com := strings.Compare(q1.Topic, q2.Topic)
+ if com != 0 {
+ return com < 0
+ }
+
+ com = strings.Compare(q1.BrokerName, q2.BrokerName)
+ if com != 0 {
+ return com < 0
+ }
+
+ return q1.QueueId < q2.QueueId
+ })
+
+ for idx := range keys {
+ dataK, err := json.Marshal(keys[idx])
+ if err != nil {
+ return nil, err
+ }
+ dataV, err := json.Marshal(info.MQTable[keys[idx]])
+ tableJson = fmt.Sprintf("%s,%s:%s", tableJson, string(dataK), string(dataV))
+ }
+ tableJson = strings.TrimLeft(tableJson, ",")
+ jsonData = fmt.Sprintf("%s,\"%s\":%s}", jsonData, "mqTable", fmt.Sprintf("{%s}", tableJson))
+ return []byte(jsonData), nil
+}
+
+func NewConsumerRunningInfo() *ConsumerRunningInfo {
+ return &ConsumerRunningInfo{
+ Properties: make(map[string]string),
+ SubscriptionData: make(map[*SubscriptionData]bool),
+ MQTable: make(map[primitive.MessageQueue]ProcessQueueInfo),
+ StatusTable: make(map[string]ConsumeStatus),
+ }
+}
diff --git a/internal/model_test.go b/internal/model_test.go
new file mode 100644
index 0000000..0d8dcd7
--- /dev/null
+++ b/internal/model_test.go
@@ -0,0 +1,364 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+ "testing"
+
+ . "github.com/smartystreets/goconvey/convey"
+ "github.com/tidwall/gjson"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func TestHeartbeatData(t *testing.T) {
+ Convey("test heatbeat json", t, func() {
+
+ Convey("producerData set marshal", func() {
+ pData := &producerData{
+ GroupName: "group name",
+ }
+ pData2 := &producerData{
+ GroupName: "group name 2",
+ }
+ set := utils.NewSet()
+ set.Add(pData)
+ set.Add(pData2)
+
+ v, err := json.Marshal(set)
+ So(err, ShouldBeNil)
+ fmt.Printf("json producer set: %s", string(v))
+ })
+
+ Convey("producer heatbeat", func() {
+
+ hbt := NewHeartbeatData("producer client id")
+ p1 := &producerData{
+ GroupName: "group name",
+ }
+ p2 := &producerData{
+ GroupName: "group name 2",
+ }
+
+ hbt.ProducerDatas.Add(p1)
+ hbt.ProducerDatas.Add(p2)
+
+ v, err := json.Marshal(hbt)
+ So(err, ShouldBeNil)
+ fmt.Printf("json producer: %s\n", string(v))
+ })
+
+ Convey("consumer heartbeat", func() {
+
+ hbt := NewHeartbeatData("consumer client id")
+ c1 := consumerData{
+ GroupName: "consumer data 1",
+ }
+ c2 := consumerData{
+ GroupName: "consumer data 2",
+ }
+ hbt.ConsumerDatas.Add(c1)
+ hbt.ConsumerDatas.Add(c2)
+
+ v, err := json.Marshal(hbt)
+ So(err, ShouldBeNil)
+ fmt.Printf("json consumer: %s\n", string(v))
+ })
+
+ Convey("producer & consumer heartbeat", func() {
+
+ hbt := NewHeartbeatData("consumer client id")
+
+ p1 := &producerData{
+ GroupName: "group name",
+ }
+ p2 := &producerData{
+ GroupName: "group name 2",
+ }
+
+ hbt.ProducerDatas.Add(p1)
+ hbt.ProducerDatas.Add(p2)
+
+ c1 := consumerData{
+ GroupName: "consumer data 1",
+ }
+ c2 := consumerData{
+ GroupName: "consumer data 2",
+ }
+ hbt.ConsumerDatas.Add(c1)
+ hbt.ConsumerDatas.Add(c2)
+
+ v, err := json.Marshal(hbt)
+ So(err, ShouldBeNil)
+ fmt.Printf("json producer & consumer: %s\n", string(v))
+ })
+ })
+
+}
+
+func TestConsumerRunningInfo_MarshalJSON(t *testing.T) {
+ Convey("test ConsumerRunningInfo MarshalJson", t, func() {
+ props := map[string]string{
+ "maxReconsumeTimes": "-1",
+ "unitMode": "false",
+ "adjustThreadPoolNumsThreshold": "100000",
+ "consumerGroup": "mq-client-go-test%GID_GO_TEST",
+ "messageModel": "CLUSTERING",
+ "suspendCurrentQueueTimeMillis": "1000",
+ "pullThresholdSizeForTopic": "-1",
+ "pullThresholdSizeForQueue": "100",
+ "PROP_CLIENT_VERSION": "V4_5_1",
+ "consumeConcurrentlyMaxSpan": "2000",
+ "postSubscriptionWhenPull": "false",
+ "consumeTimestamp": "20191127013617",
+ "PROP_CONSUME_TYPE": "CONSUME_PASSIVELY",
+ "consumeTimeout": "15",
+ "consumeMessageBatchMaxSize": "1",
+ "PROP_THREADPOOL_CORE_SIZE": "20",
+ "pullInterval": "0",
+ "pullThresholdForQueue": "1000",
+ "pullThresholdForTopic": "-1",
+ "consumeFromWhere": "CONSUME_FROM_FIRST_OFFSET",
+ "PROP_NAMESERVER_ADDR": "mq-client-go-test.mq-internet-access.mq-internet.aliyuncs.com:80;",
+ "pullBatchSize": "32",
+ "consumeThreadMin": "20",
+ "PROP_CONSUMER_START_TIMESTAMP": "1574791577504",
+ "consumeThreadMax": "20",
+ "subscription": "{}",
+ "PROP_CONSUMEORDERLY": "false",
+ }
+ subData := map[*SubscriptionData]bool{
+ &SubscriptionData{
+ ClassFilterMode: false,
+ Codes: utils.NewSet(),
+ ExpType: "TAG",
+ SubString: "*",
+ SubVersion: 1574791579242,
+ Tags: utils.NewSet(),
+ Topic: "%RETRY%mq-client-go-test%GID_GO_TEST",
+ }: true,
+ &SubscriptionData{
+ ClassFilterMode: true,
+ Codes: utils.NewSet(),
+ ExpType: "TAG",
+ SubString: "*",
+ SubVersion: 1574791577523,
+ Tags: utils.NewSet(),
+ Topic: "mq-client-go-test%go-test",
+ }: true,
+ }
+ statusTable := map[string]ConsumeStatus{
+ "%RETRY%mq-client-go-test%GID_GO_TEST": {
+ PullRT: 11.11,
+ PullTPS: 22.22,
+ ConsumeRT: 33.33,
+ ConsumeOKTPS: 44.44,
+ ConsumeFailedTPS: 55.55,
+ ConsumeFailedMsgs: 666,
+ },
+ "mq-client-go-test%go-test": {
+ PullRT: 123,
+ PullTPS: 123,
+ ConsumeRT: 123,
+ ConsumeOKTPS: 123,
+ ConsumeFailedTPS: 123,
+ ConsumeFailedMsgs: 1234,
+ },
+ }
+ mqTable := map[primitive.MessageQueue]ProcessQueueInfo{
+ {
+ Topic: "%RETRY%mq-client-go-test%GID_GO_TEST",
+ BrokerName: "qd7internet-01",
+ QueueId: 0,
+ }: {
+ CommitOffset: 0,
+ CachedMsgMinOffset: 0,
+ CachedMsgMaxOffset: 0,
+ CachedMsgCount: 0,
+ CachedMsgSizeInMiB: 0,
+ TransactionMsgMinOffset: 0,
+ TransactionMsgMaxOffset: 0,
+ TransactionMsgCount: 0,
+ Locked: false,
+ TryUnlockTimes: 0,
+ LastLockTimestamp: 1574791579221,
+ Dropped: false,
+ LastPullTimestamp: 1574791579242,
+ LastConsumeTimestamp: 1574791579221,
+ },
+ {
+ Topic: "%RETRY%mq-client-go-test%GID_GO_TEST",
+ BrokerName: "qd7internet-01",
+ QueueId: 1,
+ }: {
+ CommitOffset: 1,
+ CachedMsgMinOffset: 2,
+ CachedMsgMaxOffset: 3,
+ CachedMsgCount: 4,
+ CachedMsgSizeInMiB: 5,
+ TransactionMsgMinOffset: 6,
+ TransactionMsgMaxOffset: 7,
+ TransactionMsgCount: 8,
+ Locked: true,
+ TryUnlockTimes: 9,
+ LastLockTimestamp: 1574791579221,
+ Dropped: false,
+ LastPullTimestamp: 1574791579242,
+ LastConsumeTimestamp: 1574791579221,
+ },
+ }
+ info := ConsumerRunningInfo{
+ Properties: props,
+ SubscriptionData: subData,
+ StatusTable: statusTable,
+ MQTable: mqTable,
+ }
+ data, err := info.Encode()
+ So(err, ShouldBeNil)
+ result := gjson.ParseBytes(data)
+ Convey("test Properties fields", func() {
+ r1 := result.Get("properties")
+ So(r1.Exists(), ShouldBeTrue)
+ m := r1.Map()
+ So(len(m), ShouldEqual, 27)
+
+ So(m["PROP_CLIENT_VERSION"], ShouldNotBeEmpty)
+ So(m["PROP_CLIENT_VERSION"].String(), ShouldEqual, "V4_5_1")
+
+ So(m["PROP_CONSUME_TYPE"], ShouldNotBeNil)
+ So(m["PROP_CONSUME_TYPE"].String(), ShouldEqual, "CONSUME_PASSIVELY")
+
+ So(m["PROP_THREADPOOL_CORE_SIZE"], ShouldNotBeNil)
+ So(m["PROP_THREADPOOL_CORE_SIZE"].String(), ShouldEqual, "20")
+
+ So(m["PROP_NAMESERVER_ADDR"], ShouldNotBeNil)
+ So(m["PROP_NAMESERVER_ADDR"].String(), ShouldEqual, "mq-client-go-test.mq-internet-access.mq-internet.aliyuncs.com:80;")
+
+ So(m["PROP_CONSUMER_START_TIMESTAMP"], ShouldNotBeNil)
+ So(m["PROP_CONSUMER_START_TIMESTAMP"].String(), ShouldEqual, "1574791577504")
+
+ So(m["PROP_CONSUMEORDERLY"], ShouldNotBeNil)
+ So(m["PROP_CONSUMEORDERLY"].String(), ShouldEqual, "false")
+ })
+ Convey("test SubscriptionData fields", func() {
+ r2 := result.Get("subscriptionSet")
+ So(r2.Exists(), ShouldBeTrue)
+ arr := r2.Array()
+ So(len(arr), ShouldEqual, 2)
+
+ m1 := arr[0].Map()
+ So(len(m1), ShouldEqual, 7)
+ So(m1["classFilterMode"].Bool(), ShouldEqual, false)
+ So(len(m1["codes"].Array()), ShouldEqual, 0)
+ So(m1["expressionType"].String(), ShouldEqual, "TAG")
+ So(m1["subString"].String(), ShouldEqual, "*")
+ So(m1["subVersion"].Int(), ShouldEqual, 1574791579242)
+ So(len(m1["tags"].Array()), ShouldEqual, 0)
+ So(m1["topic"].String(), ShouldEqual, "%RETRY%mq-client-go-test%GID_GO_TEST")
+
+ m2 := arr[1].Map()
+ So(len(m2), ShouldEqual, 7)
+ So(m2["classFilterMode"].Bool(), ShouldEqual, true)
+ So(len(m2["codes"].Array()), ShouldEqual, 0)
+ So(m2["expressionType"].String(), ShouldEqual, "TAG")
+ So(m2["subString"].String(), ShouldEqual, "*")
+ So(m2["subVersion"].Int(), ShouldEqual, 1574791577523)
+ So(len(m2["tags"].Array()), ShouldEqual, 0)
+ So(m2["topic"].String(), ShouldEqual, "mq-client-go-test%go-test")
+ })
+ Convey("test StatusTable fields", func() {
+ r3 := result.Get("statusTable")
+ So(r3.Exists(), ShouldBeTrue)
+ m := r3.Map()
+ So(len(m), ShouldEqual, 2)
+
+ status1 := m["mq-client-go-test%go-test"].Map()
+ So(len(status1), ShouldEqual, 6)
+ So(status1["pullRT"].Float(), ShouldEqual, 123)
+ So(status1["pullTPS"].Float(), ShouldEqual, 123)
+ So(status1["consumeRT"].Float(), ShouldEqual, 123)
+ So(status1["consumeOKTPS"].Float(), ShouldEqual, 123)
+ So(status1["consumeFailedTPS"].Float(), ShouldEqual, 123)
+ So(status1["consumeFailedMsgs"].Int(), ShouldEqual, 1234)
+
+ status2 := m["%RETRY%mq-client-go-test%GID_GO_TEST"].Map()
+ So(len(status2), ShouldEqual, 6)
+ So(status2["pullRT"].Float(), ShouldEqual, 11.11)
+ So(status2["pullTPS"].Float(), ShouldEqual, 22.22)
+ So(status2["consumeRT"].Float(), ShouldEqual, 33.33)
+ So(status2["consumeOKTPS"].Float(), ShouldEqual, 44.44)
+ So(status2["consumeFailedTPS"].Float(), ShouldEqual, 55.55)
+ So(status2["consumeFailedMsgs"].Int(), ShouldEqual, 666)
+ })
+ Convey("test MQTable fields", func() {
+ r4 := result.Get("mqTable")
+ So(r4.Exists(), ShouldBeTrue)
+ objNumbers := strings.Split(r4.String(), "},{")
+ So(len(objNumbers), ShouldEqual, 2)
+
+ obj1Str := objNumbers[0][1:len(objNumbers[0])] + "}"
+ obj1KV := strings.Split(obj1Str, "}:{")
+ So(len(obj1KV), ShouldEqual, 2)
+
+ obj1 := gjson.Parse("{" + obj1KV[1][0:len(obj1KV[1])])
+ So(obj1.Exists(), ShouldBeTrue)
+ obj1M := obj1.Map()
+ So(len(obj1M), ShouldEqual, 14)
+ So(obj1M["commitOffset"].Int(), ShouldEqual, 0)
+ So(obj1M["cachedMsgMinOffset"].Int(), ShouldEqual, 0)
+ So(obj1M["cachedMsgMaxOffset"].Int(), ShouldEqual, 0)
+ So(obj1M["cachedMsgCount"].Int(), ShouldEqual, 0)
+ So(obj1M["cachedMsgSizeInMiB"].Int(), ShouldEqual, 0)
+ So(obj1M["transactionMsgMinOffset"].Int(), ShouldEqual, 0)
+ So(obj1M["transactionMsgMaxOffset"].Int(), ShouldEqual, 0)
+ So(obj1M["transactionMsgCount"].Int(), ShouldEqual, 0)
+ So(obj1M["locked"].Bool(), ShouldEqual, false)
+ So(obj1M["tryUnlockTimes"].Int(), ShouldEqual, 0)
+ So(obj1M["lastLockTimestamp"].Int(), ShouldEqual, 1574791579221)
+ So(obj1M["dropped"].Bool(), ShouldEqual, false)
+ So(obj1M["lastPullTimestamp"].Int(), ShouldEqual, 1574791579242)
+ So(obj1M["lastConsumeTimestamp"].Int(), ShouldEqual, 1574791579221)
+
+ obj2Str := "{" + objNumbers[1][0:len(objNumbers[1])-1]
+ obj2KV := strings.Split(obj2Str, "}:{")
+ So(len(obj2KV), ShouldEqual, 2)
+ obj2 := gjson.Parse("{" + obj2KV[1][0:len(obj2KV[1])])
+ So(obj2.Exists(), ShouldBeTrue)
+ obj2M := obj2.Map()
+ So(len(obj2M), ShouldEqual, 14)
+ So(obj2M["commitOffset"].Int(), ShouldEqual, 1)
+ So(obj2M["cachedMsgMinOffset"].Int(), ShouldEqual, 2)
+ So(obj2M["cachedMsgMaxOffset"].Int(), ShouldEqual, 3)
+ So(obj2M["cachedMsgCount"].Int(), ShouldEqual, 4)
+ So(obj2M["cachedMsgSizeInMiB"].Int(), ShouldEqual, 5)
+ So(obj2M["transactionMsgMinOffset"].Int(), ShouldEqual, 6)
+ So(obj2M["transactionMsgMaxOffset"].Int(), ShouldEqual, 7)
+ So(obj2M["transactionMsgCount"].Int(), ShouldEqual, 8)
+ So(obj2M["locked"].Bool(), ShouldEqual, true)
+ So(obj2M["tryUnlockTimes"].Int(), ShouldEqual, 9)
+ So(obj2M["lastLockTimestamp"].Int(), ShouldEqual, 1574791579221)
+ So(obj2M["dropped"].Bool(), ShouldEqual, false)
+ So(obj2M["lastPullTimestamp"].Int(), ShouldEqual, 1574791579242)
+ So(obj2M["lastConsumeTimestamp"].Int(), ShouldEqual, 1574791579221)
+ })
+ })
+}
diff --git a/internal/mq_version.go b/internal/mq_version.go
new file mode 100644
index 0000000..4b5c645
--- /dev/null
+++ b/internal/mq_version.go
@@ -0,0 +1,22 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing software
+distributed under the License is distributed on an "AS IS" BASIS
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+const (
+ V4_1_0 = 0
+)
diff --git a/internal/namesrv.go b/internal/namesrv.go
new file mode 100644
index 0000000..6d8ff2b
--- /dev/null
+++ b/internal/namesrv.go
@@ -0,0 +1,165 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "errors"
+ "regexp"
+ "strings"
+ "sync"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+const (
+ DEFAULT_NAMESRV_ADDR = "http://jmenv.tbsite.net:8080/rocketmq/nsaddr"
+)
+
+var (
+ ipRegex, _ = regexp.Compile(`^((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))`)
+
+ ErrNoNameserver = errors.New("nameServerAddrs can't be empty.")
+ ErrMultiIP = errors.New("multiple IP addr does not support")
+ ErrIllegalIP = errors.New("IP addr error")
+)
+
+//go:generate mockgen -source namesrv.go -destination mock_namesrv.go -self_package github.com/apache/rocketmq-client-go/v2/internal --package internal Namesrvs
+type Namesrvs interface {
+ UpdateNameServerAddress()
+
+ AddBroker(routeData *TopicRouteData)
+
+ cleanOfflineBroker()
+
+ UpdateTopicRouteInfo(topic string) (routeData *TopicRouteData, changed bool, err error)
+
+ FetchPublishMessageQueues(topic string) ([]*primitive.MessageQueue, error)
+
+ FindBrokerAddrByTopic(topic string) string
+
+ FindBrokerAddrByName(brokerName string) string
+
+ FindBrokerAddressInSubscribe(brokerName string, brokerId int64, onlyThisBroker bool) *FindBrokerResult
+
+ FetchSubscribeMessageQueues(topic string) ([]*primitive.MessageQueue, error)
+
+ AddrList() []string
+}
+
+// namesrvs rocketmq namesrv instance.
+type namesrvs struct {
+ // namesrv addr list
+ srvs []string
+
+ // lock for getNameServerAddress in case of update index race condition
+ lock sync.Locker
+
+ // index indicate the next position for getNameServerAddress
+ index int
+
+ // brokerName -> *BrokerData
+ brokerAddressesMap sync.Map
+
+ // brokerName -> map[string]int32: brokerAddr -> version
+ brokerVersionMap map[string]map[string]int32
+ // lock for broker version read/write
+ brokerLock *sync.RWMutex
+
+ //subscribeInfoMap sync.Map
+ routeDataMap sync.Map
+
+ lockNamesrv sync.Mutex
+
+ nameSrvClient remote.RemotingClient
+
+ resolver primitive.NsResolver
+}
+
+var _ Namesrvs = (*namesrvs)(nil)
+
+// NewNamesrv init Namesrv from namesrv addr string.
+// addr primitive.NamesrvAddr
+func NewNamesrv(resolver primitive.NsResolver) (*namesrvs, error) {
+ addr := resolver.Resolve()
+ if len(addr) == 0 {
+ return nil, errors.New("no name server addr found with resolver: " + resolver.Description())
+ }
+
+ if err := primitive.NamesrvAddr(addr).Check(); err != nil {
+ return nil, err
+ }
+ nameSrvClient := remote.NewRemotingClient()
+ return &namesrvs{
+ srvs: addr,
+ lock: new(sync.Mutex),
+ nameSrvClient: nameSrvClient,
+ brokerVersionMap: make(map[string]map[string]int32, 0),
+ brokerLock: new(sync.RWMutex),
+ resolver: resolver,
+ }, nil
+}
+
+// getNameServerAddress return namesrv using round-robin strategy.
+func (s *namesrvs) getNameServerAddress() string {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ addr := s.srvs[s.index]
+ index := s.index + 1
+ if index < 0 {
+ index = -index
+ }
+ index %= len(s.srvs)
+ s.index = index
+ return strings.TrimLeft(addr, "http(s)://")
+}
+
+func (s *namesrvs) Size() int {
+ return len(s.srvs)
+}
+
+func (s *namesrvs) String() string {
+ return strings.Join(s.srvs, ";")
+}
+func (s *namesrvs) SetCredentials(credentials primitive.Credentials) {
+ s.nameSrvClient.RegisterInterceptor(remote.ACLInterceptor(credentials))
+}
+
+func (s *namesrvs) AddrList() []string {
+ return s.srvs
+}
+
+// UpdateNameServerAddress will update srvs.
+// docs: https://rocketmq.apache.org/docs/best-practice-namesvr/
+func (s *namesrvs) UpdateNameServerAddress() {
+ s.lock.Lock()
+ defer s.lock.Unlock()
+
+ srvs := s.resolver.Resolve()
+ if len(srvs) == 0 {
+ return
+ }
+
+ updated := primitive.Diff(s.srvs, srvs)
+ if !updated {
+ return
+ }
+
+ s.srvs = srvs
+}
diff --git a/internal/namesrv_test.go b/internal/namesrv_test.go
new file mode 100644
index 0000000..e58dc29
--- /dev/null
+++ b/internal/namesrv_test.go
@@ -0,0 +1,146 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "fmt"
+ "net"
+ "net/http"
+ "os"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+
+ . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
+)
+
+// TestSelector test roundrobin selector in namesrv
+func TestSelector(t *testing.T) {
+ srvs := []string{"127.0.0.1:9876", "127.0.0.1:9879", "12.24.123.243:10911", "12.24.123.243:10915"}
+ namesrv, err := NewNamesrv(primitive.NewPassthroughResolver(srvs))
+ assert.Nil(t, err)
+
+ assert.Equal(t, srvs[0], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[1], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[2], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[3], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[0], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[1], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[2], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[3], namesrv.getNameServerAddress())
+ assert.Equal(t, srvs[0], namesrv.getNameServerAddress())
+}
+
+func TestGetNamesrv(t *testing.T) {
+ Convey("Test GetNamesrv round-robin strategy", t, func() {
+ ns := &namesrvs{
+ srvs: []string{"192.168.100.1",
+ "192.168.100.2",
+ "192.168.100.3",
+ "192.168.100.4",
+ "192.168.100.5",
+ },
+ lock: new(sync.Mutex),
+ }
+
+ index1 := ns.index
+ IP1 := ns.getNameServerAddress()
+
+ index2 := ns.index
+ IP2 := ns.getNameServerAddress()
+
+ So(index1+1, ShouldEqual, index2)
+ So(IP1, ShouldEqual, ns.srvs[index1])
+ So(IP2, ShouldEqual, ns.srvs[index2])
+ })
+}
+
+func TestUpdateNameServerAddress(t *testing.T) {
+ Convey("Test UpdateNameServerAddress method", t, func() {
+ srvs := []string{
+ "192.168.100.1",
+ "192.168.100.2",
+ "192.168.100.3",
+ "192.168.100.4",
+ "192.168.100.5",
+ }
+ http.HandleFunc("/nameserver/addrs", func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprintf(w, strings.Join(srvs, ";"))
+ })
+ server := &http.Server{Addr: ":0", Handler: nil}
+ listener, _ := net.Listen("tcp", ":0")
+ go server.Serve(listener)
+
+ port := listener.Addr().(*net.TCPAddr).Port
+ nameServerDommain := fmt.Sprintf("http://127.0.0.1:%d/nameserver/addrs", port)
+ fmt.Println("temporary name server domain: ", nameServerDommain)
+
+ resolver := primitive.NewHttpResolver("DEFAULT", nameServerDommain)
+ ns := &namesrvs{
+ srvs: []string{},
+ lock: new(sync.Mutex),
+ resolver: resolver,
+ }
+
+ ns.UpdateNameServerAddress()
+
+ index1 := ns.index
+ IP1 := ns.getNameServerAddress()
+
+ index2 := ns.index
+ IP2 := ns.getNameServerAddress()
+
+ So(index1+1, ShouldEqual, index2)
+ So(IP1, ShouldEqual, srvs[index1])
+ So(IP2, ShouldEqual, srvs[index2])
+ })
+}
+
+func TestUpdateNameServerAddressUseEnv(t *testing.T) {
+ Convey("Test UpdateNameServerAddress Use Env", t, func() {
+ srvs := []string{
+ "192.168.100.1",
+ "192.168.100.2",
+ "192.168.100.3",
+ "192.168.100.4",
+ "192.168.100.5",
+ }
+
+ resolver := primitive.NewEnvResolver()
+ ns := &namesrvs{
+ srvs: []string{},
+ lock: new(sync.Mutex),
+ resolver: resolver,
+ }
+ os.Setenv("NAMESRV_ADDR", strings.Join(srvs, ";"))
+ ns.UpdateNameServerAddress()
+
+ index1 := ns.index
+ IP1 := ns.getNameServerAddress()
+
+ index2 := ns.index
+ IP2 := ns.getNameServerAddress()
+
+ So(index1+1, ShouldEqual, index2)
+ So(IP1, ShouldEqual, srvs[index1])
+ So(IP2, ShouldEqual, srvs[index2])
+ })
+}
diff --git a/internal/perm.go b/internal/perm.go
new file mode 100644
index 0000000..638b7a7
--- /dev/null
+++ b/internal/perm.go
@@ -0,0 +1,58 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+const (
+ permPriority = 0x1 << 3
+ permRead = 0x1 << 2
+ permWrite = 0x1 << 1
+ permInherit = 0x1 << 0
+)
+
+func queueIsReadable(perm int) bool {
+ return (perm & permRead) == permRead
+}
+
+func queueIsWriteable(perm int) bool {
+ return (perm & permWrite) == permWrite
+}
+
+func queueIsInherited(perm int) bool {
+ return (perm & permInherit) == permInherit
+}
+
+func perm2string(perm int) string {
+ bytes := make([]byte, 3)
+ for i := 0; i < 3; i++ {
+ bytes[i] = '-'
+ }
+
+ if queueIsReadable(perm) {
+ bytes[0] = 'R'
+ }
+
+ if queueIsWriteable(perm) {
+ bytes[1] = 'W'
+ }
+
+ if queueIsInherited(perm) {
+ bytes[2] = 'X'
+ }
+
+ return string(bytes)
+}
diff --git a/internal/remote/codec.go b/internal/remote/codec.go
new file mode 100644
index 0000000..f756c11
--- /dev/null
+++ b/internal/remote/codec.go
@@ -0,0 +1,488 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package remote
+
+import (
+ "bytes"
+ "encoding/binary"
+ "fmt"
+ "sync/atomic"
+
+ jsoniter "github.com/json-iterator/go"
+)
+
+var opaque int32
+
+const (
+ // 0, REQUEST_COMMAND
+ RPCType = 0
+ // 1, RPC
+ RPCOneWay = 1
+ //ResponseType for response
+ ResponseType = 1
+ _Flag = 0
+ _Version = 317
+)
+
+type LanguageCode byte
+
+const (
+ _Java = LanguageCode(0)
+ _Go = LanguageCode(9)
+ _Unknown = LanguageCode(127)
+)
+
+func (lc LanguageCode) MarshalJSON() ([]byte, error) {
+ return []byte(`"GO"`), nil
+}
+
+func (lc *LanguageCode) UnmarshalJSON(b []byte) error {
+ switch string(b) {
+ case "JAVA":
+ *lc = _Java
+ case "GO", `"GO"`:
+ *lc = _Go
+ default:
+ *lc = _Unknown
+ }
+ return nil
+}
+
+func (lc LanguageCode) String() string {
+ switch lc {
+ case _Java:
+ return "JAVA"
+ case _Go:
+ return "GO"
+ default:
+ return "unknown"
+ }
+}
+
+type RemotingCommand struct {
+ Code int16 `json:"code"`
+ Language LanguageCode `json:"language"`
+ Version int16 `json:"version"`
+ Opaque int32 `json:"opaque"`
+ Flag int32 `json:"flag"`
+ Remark string `json:"remark"`
+ ExtFields map[string]string `json:"extFields"`
+ Body []byte `json:"-"`
+}
+
+type CustomHeader interface {
+ Encode() map[string]string
+}
+
+func NewRemotingCommand(code int16, header CustomHeader, body []byte) *RemotingCommand {
+ cmd := &RemotingCommand{
+ Code: code,
+ Version: _Version,
+ Opaque: atomic.AddInt32(&opaque, 1),
+ Body: body,
+ Language: _Go,
+ ExtFields: make(map[string]string),
+ }
+
+ if header != nil {
+ cmd.ExtFields = header.Encode()
+ }
+
+ return cmd
+}
+
+func (command *RemotingCommand) String() string {
+ return fmt.Sprintf("Code: %d, opaque: %d, Remark: %s, ExtFields: %v",
+ command.Code, command.Opaque, command.Remark, command.ExtFields)
+}
+
+func (command *RemotingCommand) isResponseType() bool {
+ return command.Flag&(ResponseType) == ResponseType
+}
+
+func (command *RemotingCommand) markResponseType() {
+ command.Flag = command.Flag | ResponseType
+}
+
+var (
+ jsonSerializer = &jsonCodec{}
+ rocketMqSerializer = &rmqCodec{}
+ codecType byte
+)
+
+// encode RemotingCommand
+//
+// Frame format:
+// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+// + item | frame_size | header_length | header_body | body +
+// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+// + len | 4bytes | 4bytes | (21 + r_len + e_len) bytes | remain bytes +
+// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+func encode(command *RemotingCommand) ([]byte, error) {
+ var (
+ header []byte
+ err error
+ )
+
+ switch codecType {
+ case JsonCodecs:
+ header, err = jsonSerializer.encodeHeader(command)
+ case RocketMQCodecs:
+ header, err = rocketMqSerializer.encodeHeader(command)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+
+ frameSize := 4 + len(header) + len(command.Body)
+ buf := bytes.NewBuffer(make([]byte, frameSize))
+ buf.Reset()
+
+ err = binary.Write(buf, binary.BigEndian, int32(frameSize))
+ if err != nil {
+ return nil, err
+ }
+
+ err = binary.Write(buf, binary.BigEndian, markProtocolType(int32(len(header))))
+ if err != nil {
+ return nil, err
+ }
+
+ err = binary.Write(buf, binary.BigEndian, header)
+ if err != nil {
+ return nil, err
+ }
+
+ err = binary.Write(buf, binary.BigEndian, command.Body)
+ if err != nil {
+ return nil, err
+ }
+
+ return buf.Bytes(), nil
+}
+
+func decode(data []byte) (*RemotingCommand, error) {
+ buf := bytes.NewBuffer(data)
+ length := int32(len(data))
+ var oriHeaderLen int32
+ err := binary.Read(buf, binary.BigEndian, &oriHeaderLen)
+ if err != nil {
+ return nil, err
+ }
+
+ headerLength := oriHeaderLen & 0xFFFFFF
+ headerData := make([]byte, headerLength)
+ err = binary.Read(buf, binary.BigEndian, &headerData)
+ if err != nil {
+ return nil, err
+ }
+
+ var command *RemotingCommand
+ switch codeType := byte((oriHeaderLen >> 24) & 0xFF); codeType {
+ case JsonCodecs:
+ command, err = jsonSerializer.decodeHeader(headerData)
+ case RocketMQCodecs:
+ command, err = rocketMqSerializer.decodeHeader(headerData)
+ default:
+ err = fmt.Errorf("unknown codec type: %d", codeType)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ bodyLength := length - 4 - headerLength
+ if bodyLength > 0 {
+ bodyData := make([]byte, bodyLength)
+ err = binary.Read(buf, binary.BigEndian, &bodyData)
+ if err != nil {
+ return nil, err
+ }
+ command.Body = bodyData
+ }
+ return command, nil
+}
+
+func markProtocolType(source int32) []byte {
+ result := make([]byte, 4)
+ result[0] = codecType
+ result[1] = byte((source >> 16) & 0xFF)
+ result[2] = byte((source >> 8) & 0xFF)
+ result[3] = byte(source & 0xFF)
+ return result
+}
+
+const (
+ JsonCodecs = byte(0)
+ RocketMQCodecs = byte(1)
+)
+
+type serializer interface {
+ encodeHeader(command *RemotingCommand) ([]byte, error)
+ decodeHeader(data []byte) (*RemotingCommand, error)
+}
+
+// jsonCodec please refer to remoting/protocol/RemotingSerializable
+type jsonCodec struct{}
+
+func (c *jsonCodec) encodeHeader(command *RemotingCommand) ([]byte, error) {
+ buf, err := jsoniter.Marshal(command)
+ if err != nil {
+ return nil, err
+ }
+ return buf, nil
+}
+
+func (c *jsonCodec) decodeHeader(header []byte) (*RemotingCommand, error) {
+ command := &RemotingCommand{}
+ command.ExtFields = make(map[string]string)
+ command.Body = make([]byte, 0)
+ err := jsoniter.Unmarshal(header, command)
+ if err != nil {
+ return nil, err
+ }
+ return command, nil
+}
+
+// rmqCodec implementation of RocketMQCodecs private protocol, please refer to remoting/protocol/RocketMQSerializable
+// RocketMQCodecs Private Protocol Header format:
+//
+// v_flag: version flag
+// r_len: length of remark body
+// r_body: data of remark body
+// e_len: length of extends fields body
+// e_body: data of extends fields
+//
+// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+// + item | request_code | l_flag | v_flag | opaque | request_flag | r_len | r_body | e_len | e_body +
+// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+// + len | 2bytes | 1byte | 2bytes | 4bytes | 4 bytes | 4 bytes | r_len bytes | 4 bytes | e_len bytes +
+// +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+const (
+ // header + body length
+ headerFixedLength = 21
+)
+
+type rmqCodec struct{}
+
+// encodeHeader
+func (c *rmqCodec) encodeHeader(command *RemotingCommand) ([]byte, error) {
+ extBytes, err := c.encodeMaps(command.ExtFields)
+ if err != nil {
+ return nil, err
+ }
+
+ buf := bytes.NewBuffer(make([]byte, headerFixedLength+len(command.Remark)+len(extBytes)))
+ buf.Reset()
+
+ // request code, length is 2 bytes
+ err = binary.Write(buf, binary.BigEndian, int16(command.Code))
+ if err != nil {
+ return nil, err
+ }
+
+ // language flag, length is 1 byte
+ err = binary.Write(buf, binary.BigEndian, _Go)
+ if err != nil {
+ return nil, err
+ }
+
+ // version flag, length is 2 bytes
+ err = binary.Write(buf, binary.BigEndian, int16(command.Version))
+ if err != nil {
+ return nil, err
+ }
+
+ // opaque flag, opaque is request identifier, length is 4 bytes
+ err = binary.Write(buf, binary.BigEndian, command.Opaque)
+ if err != nil {
+ return nil, err
+ }
+
+ // request flag, length is 4 bytes
+ err = binary.Write(buf, binary.BigEndian, command.Flag)
+ if err != nil {
+ return nil, err
+ }
+
+ // remark length flag, length is 4 bytes
+ err = binary.Write(buf, binary.BigEndian, int32(len(command.Remark)))
+ if err != nil {
+ return nil, err
+ }
+
+ // write remark, len(command.Remark) bytes
+ if len(command.Remark) > 0 {
+ err = binary.Write(buf, binary.BigEndian, []byte(command.Remark))
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ err = binary.Write(buf, binary.BigEndian, int32(len(extBytes)))
+ if err != nil {
+ return nil, err
+ }
+
+ if len(extBytes) > 0 {
+ err = binary.Write(buf, binary.BigEndian, extBytes)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return buf.Bytes(), nil
+}
+
+func (c *rmqCodec) encodeMaps(maps map[string]string) ([]byte, error) {
+ if maps == nil || len(maps) == 0 {
+ return []byte{}, nil
+ }
+ extFieldsBuf := bytes.NewBuffer([]byte{})
+ var err error
+ for key, value := range maps {
+ err = binary.Write(extFieldsBuf, binary.BigEndian, int16(len(key)))
+ if err != nil {
+ return nil, err
+ }
+ err = binary.Write(extFieldsBuf, binary.BigEndian, []byte(key))
+ if err != nil {
+ return nil, err
+ }
+
+ err = binary.Write(extFieldsBuf, binary.BigEndian, int32(len(value)))
+ if err != nil {
+ return nil, err
+ }
+ err = binary.Write(extFieldsBuf, binary.BigEndian, []byte(value))
+ if err != nil {
+ return nil, err
+ }
+ }
+ return extFieldsBuf.Bytes(), nil
+}
+
+func (c *rmqCodec) decodeHeader(data []byte) (*RemotingCommand, error) {
+ var err error
+ command := &RemotingCommand{}
+ buf := bytes.NewBuffer(data)
+ var code int16
+ err = binary.Read(buf, binary.BigEndian, &code)
+ if err != nil {
+ return nil, err
+ }
+ command.Code = code
+
+ var (
+ languageCode byte
+ remarkLen int32
+ extFieldsLen int32
+ )
+ err = binary.Read(buf, binary.BigEndian, &languageCode)
+ if err != nil {
+ return nil, err
+ }
+ command.Language = LanguageCode(languageCode)
+
+ var version int16
+ err = binary.Read(buf, binary.BigEndian, &version)
+ if err != nil {
+ return nil, err
+ }
+ command.Version = version
+
+ // int opaque
+ err = binary.Read(buf, binary.BigEndian, &command.Opaque)
+ if err != nil {
+ return nil, err
+ }
+
+ // int flag
+ err = binary.Read(buf, binary.BigEndian, &command.Flag)
+ if err != nil {
+ return nil, err
+ }
+
+ // String remark
+ err = binary.Read(buf, binary.BigEndian, &remarkLen)
+ if err != nil {
+ return nil, err
+ }
+
+ if remarkLen > 0 {
+ var remarkData = make([]byte, remarkLen)
+ err = binary.Read(buf, binary.BigEndian, &remarkData)
+ if err != nil {
+ return nil, err
+ }
+ command.Remark = string(remarkData)
+ }
+
+ err = binary.Read(buf, binary.BigEndian, &extFieldsLen)
+ if err != nil {
+ return nil, err
+ }
+
+ if extFieldsLen > 0 {
+ extFieldsData := make([]byte, extFieldsLen)
+ err = binary.Read(buf, binary.BigEndian, &extFieldsData)
+ if err != nil {
+ return nil, err
+ }
+
+ command.ExtFields = make(map[string]string)
+ buf := bytes.NewBuffer(extFieldsData)
+ var (
+ kLen int16
+ vLen int32
+ )
+ for buf.Len() > 0 {
+ err = binary.Read(buf, binary.BigEndian, &kLen)
+ if err != nil {
+ return nil, err
+ }
+
+ key, err := getExtFieldsData(buf, int32(kLen))
+ if err != nil {
+ return nil, err
+ }
+
+ err = binary.Read(buf, binary.BigEndian, &vLen)
+ if err != nil {
+ return nil, err
+ }
+
+ value, err := getExtFieldsData(buf, vLen)
+ if err != nil {
+ return nil, err
+ }
+ command.ExtFields[key] = value
+ }
+ }
+
+ return command, nil
+}
+
+func getExtFieldsData(buff *bytes.Buffer, length int32) (string, error) {
+ var data = make([]byte, length)
+ err := binary.Read(buff, binary.BigEndian, &data)
+ if err != nil {
+ return "", err
+ }
+
+ return string(data), nil
+}
diff --git a/internal/remote/codec_test.go b/internal/remote/codec_test.go
new file mode 100644
index 0000000..8fb8a60
--- /dev/null
+++ b/internal/remote/codec_test.go
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package remote
+
+import (
+ "encoding/json"
+ "fmt"
+ "math/rand"
+ "reflect"
+ "testing"
+ "unsafe"
+
+ jsoniter "github.com/json-iterator/go"
+
+ "github.com/stretchr/testify/assert"
+)
+
+type testHeader struct {
+}
+
+func (t testHeader) Encode() map[string]string {
+ properties := make(map[string]string)
+ for i := 0; i < 10; i++ {
+ properties[randomString(rand.Intn(20))] = randomString(rand.Intn(20))
+ }
+ return properties
+}
+
+func randomBytes(length int) []byte {
+ bs := make([]byte, length)
+ if _, err := rand.Read(bs); err != nil {
+ panic("read random bytes fail")
+ }
+ return bs
+}
+
+func randomString(length int) string {
+ bs := make([]byte, length)
+ for i := 0; i < len(bs); i++ {
+ bs[i] = byte(97 + rand.Intn(26))
+ }
+ return string(bs)
+}
+
+func randomNewRemotingCommand() *RemotingCommand {
+ var h testHeader
+ body := randomBytes(rand.Intn(100))
+ return NewRemotingCommand(int16(rand.Intn(1000)), h, body)
+}
+
+func Test_encode(t *testing.T) {
+ for i := 0; i < 1000; i++ {
+ rc := randomNewRemotingCommand()
+ if _, err := encode(rc); err != nil {
+ t.Fatalf("encode RemotingCommand to bytes fail: %v", err)
+ }
+ }
+}
+
+func Benchmark_encode(b *testing.B) {
+ rc := randomNewRemotingCommand()
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ if _, err := encode(rc); err != nil {
+ b.Fatalf("encode RemotingCommand to bytes fail: %v", err)
+ }
+ }
+}
+
+func Test_decode(t *testing.T) {
+ for i := 0; i < 1000; i++ {
+ rc := randomNewRemotingCommand()
+
+ bs, err := encode(rc)
+ if err != nil {
+ t.Fatalf("encode RemotingCommand to bytes fail: %v", err)
+ }
+ bs = bs[4:]
+ decodedRc, err := decode(bs)
+ if err != nil {
+ t.Fatalf("decode bytes to RemotingCommand fail: %v", err)
+ }
+
+ if rc.Code != decodedRc.Code {
+ t.Fatalf("wrong Code. want=%d, got=%d", rc.Code, decodedRc.Code)
+ }
+ if rc.Version != decodedRc.Version {
+ t.Fatalf("wrong Version. want=%d, got=%d", rc.Version, decodedRc.Version)
+ }
+ if rc.Opaque != decodedRc.Opaque {
+ t.Fatalf("wrong opaque. want=%d, got=%d", rc.Opaque, decodedRc.Opaque)
+ }
+ if rc.Remark != decodedRc.Remark {
+ t.Fatalf("wrong remark. want=%s, got=%s", rc.Remark, decodedRc.Remark)
+ }
+ if rc.Flag != decodedRc.Flag {
+ t.Fatalf("wrong flag. want=%d, got=%d", rc.Flag, decodedRc.Flag)
+ }
+ if !reflect.DeepEqual(rc.ExtFields, decodedRc.ExtFields) {
+ t.Fatalf("wrong extFields, want=%v, got=%v", rc.ExtFields, decodedRc.ExtFields)
+ }
+ }
+}
+
+func Benchmark_decode(b *testing.B) {
+ rc := randomNewRemotingCommand()
+ bs, err := encode(rc)
+ if err != nil {
+ b.Fatalf("encode RemotingCommand to bytes fail: %v", err)
+ }
+ b.ResetTimer()
+ bs = bs[4:]
+ for i := 0; i < b.N; i++ {
+ if _, err := decode(bs); err != nil {
+ b.Fatalf("decode bytes to RemotingCommand fail: %v", err)
+ }
+ }
+}
+
+func Test_jsonCodec_encodeHeader(t *testing.T) {
+ for i := 0; i < 1000; i++ {
+ rc := randomNewRemotingCommand()
+
+ if _, err := jsonSerializer.encodeHeader(rc); err != nil {
+ t.Fatalf("encode header with jsonCodec fail: %v", err)
+ }
+ }
+}
+
+func Benchmark_jsonCodec_encodeHeader(b *testing.B) {
+ rc := randomNewRemotingCommand()
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ if _, err := jsonSerializer.encodeHeader(rc); err != nil {
+ b.Fatalf("encode header with jsonCodec fail: %v", err)
+ }
+ }
+}
+
+func Test_jsonCodec_decodeHeader(t *testing.T) {
+ for i := 0; i < 1; i++ {
+ rc := randomNewRemotingCommand()
+
+ headers, err := jsonSerializer.encodeHeader(rc)
+ if err != nil {
+ t.Fatalf("encode header with jsonCodec fail: %v", err)
+ }
+
+ decodedRc, err := jsonSerializer.decodeHeader(headers)
+ if err != nil {
+ t.Fatalf("decode header with jsonCodec fail: %v", err)
+ }
+
+ if rc.Code != decodedRc.Code {
+ t.Fatalf("wrong Code. want=%d, got=%d", rc.Code, decodedRc.Code)
+ }
+ if rc.Version != decodedRc.Version {
+ t.Fatalf("wrong Version. want=%d, got=%d", rc.Version, decodedRc.Version)
+ }
+ if rc.Opaque != decodedRc.Opaque {
+ t.Fatalf("wrong opaque. want=%d, got=%d", rc.Opaque, decodedRc.Opaque)
+ }
+ if rc.Remark != decodedRc.Remark {
+ t.Fatalf("wrong remark. want=%s, got=%s", rc.Remark, decodedRc.Remark)
+ }
+ if rc.Flag != decodedRc.Flag {
+ t.Fatalf("wrong flag. want=%d, got=%d", rc.Flag, decodedRc.Flag)
+ }
+ if !reflect.DeepEqual(rc.ExtFields, decodedRc.ExtFields) {
+ t.Fatalf("wrong extFields, want=%v, got=%v", rc.ExtFields, decodedRc.ExtFields)
+ }
+ }
+}
+
+func Benchmark_jsonCodec_decodeHeader(b *testing.B) {
+ rc := randomNewRemotingCommand()
+ headers, err := jsonSerializer.encodeHeader(rc)
+ if err != nil {
+ b.Fatalf("encode header with jsonCodec fail: %v", err)
+ }
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ if _, err := jsonSerializer.decodeHeader(headers); err != nil {
+ b.Fatalf("decode header with jsonCodec fail: %v", err)
+ }
+ }
+}
+
+func Test_rmqCodec_encodeHeader(t *testing.T) {
+ for i := 0; i < 1000; i++ {
+ rc := randomNewRemotingCommand()
+
+ if _, err := rocketMqSerializer.encodeHeader(rc); err != nil {
+ t.Fatalf("encode header with rmqCodec fail: %v", err)
+ }
+ }
+}
+
+func Benchmark_rmqCodec_encodeHeader(b *testing.B) {
+ rc := randomNewRemotingCommand()
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ if _, err := rocketMqSerializer.encodeHeader(rc); err != nil {
+ b.Fatalf("encode header with rmqCodec fail: %v", err)
+ }
+ }
+}
+
+func Test_rmqCodec_decodeHeader(t *testing.T) {
+ for i := 0; i < 1; i++ {
+ rc := randomNewRemotingCommand()
+
+ headers, err := rocketMqSerializer.encodeHeader(rc)
+ if err != nil {
+ t.Fatalf("encode header with rmqCodec fail: %v", err)
+ }
+
+ decodedRc, err := rocketMqSerializer.decodeHeader(headers)
+ if err != nil {
+ t.Fatalf("decode header with rmqCodec fail: %v", err)
+ }
+ if rc.Code != decodedRc.Code {
+ t.Fatalf("wrong Code. want=%d, got=%d", rc.Code, decodedRc.Code)
+ }
+ if rc.Version != decodedRc.Version {
+ t.Fatalf("wrong Version. want=%d, got=%d", rc.Version, decodedRc.Version)
+ }
+ if rc.Opaque != decodedRc.Opaque {
+ t.Fatalf("wrong opaque. want=%d, got=%d", rc.Opaque, decodedRc.Opaque)
+ }
+ if rc.Remark != decodedRc.Remark {
+ t.Fatalf("wrong remark. want=%s, got=%s", rc.Remark, decodedRc.Remark)
+ }
+ if rc.Flag != decodedRc.Flag {
+ t.Fatalf("wrong flag. want=%d, got=%d", rc.Flag, decodedRc.Flag)
+ }
+ if !reflect.DeepEqual(rc.ExtFields, decodedRc.ExtFields) {
+ t.Fatalf("wrong extFields, want=%v, got=%v", rc.ExtFields, decodedRc.ExtFields)
+ }
+
+ }
+}
+
+func Benchmark_rmqCodec_decodeHeader(b *testing.B) {
+ rc := randomNewRemotingCommand()
+ headers, err := rocketMqSerializer.encodeHeader(rc)
+ if err != nil {
+ b.Fatalf("encode header with rmqCodec fail: %v", err)
+ }
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ if _, err := rocketMqSerializer.decodeHeader(headers); err != nil {
+ b.Fatalf("decode header with rmqCodec fail: %v", err)
+ }
+ }
+}
+
+func TestCommandJsonEncodeDecode(t *testing.T) {
+ var h testHeader
+ cmd := NewRemotingCommand(192, h, []byte("Hello RocketMQCodecs"))
+ codecType = JsonCodecs
+ cmdData, err := encode(cmd)
+ if err != nil {
+ t.Errorf("failed to encode remotingCommand in JSON, %s", err)
+ } else {
+ if len(cmdData) == 0 {
+ t.Errorf("failed to encode remotingCommand, result is empty.")
+ }
+ }
+ cmdData = cmdData[4:]
+ newCmd, err := decode(cmdData)
+ if err != nil {
+ t.Errorf("failed to decode remoting in JSON. %s", err)
+ }
+ if newCmd.Code != cmd.Code {
+ t.Errorf("wrong command code. want=%d, got=%d", cmd.Code, newCmd.Code)
+ }
+ if newCmd.Version != cmd.Version {
+ t.Errorf("wrong command version. want=%d, got=%d", cmd.Version, newCmd.Version)
+ }
+ if newCmd.Opaque != cmd.Opaque {
+ t.Errorf("wrong command version. want=%d, got=%d", cmd.Opaque, newCmd.Opaque)
+ }
+ if newCmd.Flag != cmd.Flag {
+ t.Errorf("wrong commad flag. want=%d, got=%d", cmd.Flag, newCmd.Flag)
+ }
+ if newCmd.Remark != cmd.Remark {
+ t.Errorf("wrong command remakr. want=%s, got=%s", cmd.Remark, newCmd.Remark)
+ }
+}
+
+func TestCommandRocketMQEncodeDecode(t *testing.T) {
+ var h testHeader
+ cmd := NewRemotingCommand(192, h, []byte("Hello RocketMQCodecs"))
+ codecType = RocketMQCodecs
+ cmdData, err := encode(cmd)
+ if err != nil {
+ t.Errorf("failed to encode remotingCommand in JSON, %s", err)
+ } else {
+ if len(cmdData) == 0 {
+ t.Errorf("failed to encode remotingCommand, result is empty.")
+ }
+ }
+ cmdData = cmdData[4:]
+ newCmd, err := decode(cmdData)
+ if err != nil {
+ t.Errorf("failed to decode remoting in JSON. %s", err)
+ }
+ if newCmd.Code != cmd.Code {
+ t.Errorf("wrong command code. want=%d, got=%d", cmd.Code, newCmd.Code)
+ }
+ if newCmd.Language != cmd.Language {
+ t.Errorf("wrong command language. want=%d, got=%d", cmd.Language, newCmd.Language)
+ }
+ if newCmd.Version != cmd.Version {
+ t.Errorf("wrong command version. want=%d, got=%d", cmd.Version, newCmd.Version)
+ }
+ if newCmd.Opaque != cmd.Opaque {
+ t.Errorf("wrong command version. want=%d, got=%d", cmd.Opaque, newCmd.Opaque)
+ }
+ if newCmd.Flag != cmd.Flag {
+ t.Errorf("wrong commad flag. want=%d, got=%d", cmd.Flag, newCmd.Flag)
+ }
+ if newCmd.Remark != cmd.Remark {
+ t.Errorf("wrong command remakr. want=%s, got=%s", cmd.Remark, newCmd.Remark)
+ }
+}
+
+func TestCommandJsonIter(t *testing.T) {
+ var h testHeader
+ cmd := NewRemotingCommand(192, h, []byte("Hello RocketMQCodecs"))
+ cmdData, err := json.Marshal(cmd)
+ assert.Nil(t, err)
+ fmt.Printf("cmd data from json: %v\n", *(*string)(unsafe.Pointer(&cmdData)))
+
+ data, err := jsoniter.Marshal(cmd)
+ assert.Nil(t, err)
+ fmt.Printf("cmd data from jsoniter: %v\n", *(*string)(unsafe.Pointer(&data)))
+
+ var cmdResp RemotingCommand
+ err = json.Unmarshal(cmdData, &cmdResp)
+ assert.Nil(t, err)
+ fmt.Printf("cmd: %#v language: %v\n", cmdResp, cmdResp.Language)
+
+ var cmdResp2 RemotingCommand
+ err = json.Unmarshal(data, &cmdResp2)
+ assert.Nil(t, err)
+ fmt.Printf("cmd: %#v language: %v\n", cmdResp2, cmdResp2.Language)
+}
diff --git a/internal/remote/future.go b/internal/remote/future.go
new file mode 100644
index 0000000..ffbf781
--- /dev/null
+++ b/internal/remote/future.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+ "context"
+ "sync"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+)
+
+// ResponseFuture
+type ResponseFuture struct {
+ ResponseCommand *RemotingCommand
+ Err error
+ Opaque int32
+ callback func(*ResponseFuture)
+ Done chan bool
+ callbackOnce sync.Once
+ ctx context.Context
+}
+
+// NewResponseFuture create ResponseFuture with opaque, timeout and callback
+func NewResponseFuture(ctx context.Context, opaque int32, callback func(*ResponseFuture)) *ResponseFuture {
+ return &ResponseFuture{
+ Opaque: opaque,
+ Done: make(chan bool),
+ callback: callback,
+ ctx: ctx,
+ }
+}
+
+func (r *ResponseFuture) executeInvokeCallback() {
+ r.callbackOnce.Do(func() {
+ if r.callback != nil {
+ r.callback(r)
+ }
+ })
+}
+
+func (r *ResponseFuture) waitResponse() (*RemotingCommand, error) {
+ var (
+ cmd *RemotingCommand
+ err error
+ )
+ select {
+ case <-r.Done:
+ cmd, err = r.ResponseCommand, r.Err
+ case <-r.ctx.Done():
+ err = utils.ErrRequestTimeout
+ r.Err = err
+ }
+ return cmd, err
+}
diff --git a/internal/remote/interceptor.go b/internal/remote/interceptor.go
new file mode 100644
index 0000000..cf96717
--- /dev/null
+++ b/internal/remote/interceptor.go
@@ -0,0 +1,83 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+ "context"
+ "crypto/hmac"
+ "crypto/sha1"
+ "encoding/base64"
+ "hash"
+ "sort"
+ "strings"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+const (
+ signature = "Signature"
+ accessKey = "AccessKey"
+ securityToken = "SecurityToken"
+ keyFile = "KEY_FILE"
+ // System.getProperty("rocketmq.client.keyFile", System.getProperty("user.home") + File.separator + "key");
+)
+
+func ACLInterceptor(credentials primitive.Credentials) primitive.Interceptor {
+ return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
+ cmd := req.(*RemotingCommand)
+ m := make(map[string]string)
+ order := make([]string, 1)
+ m[accessKey] = credentials.AccessKey
+ order[0] = accessKey
+ if credentials.SecurityToken != "" {
+ m[securityToken] = credentials.SecurityToken
+ }
+ for k, v := range cmd.ExtFields {
+ m[k] = v
+ order = append(order, k)
+ }
+ sort.Slice(order, func(i, j int) bool {
+ return strings.Compare(order[i], order[j]) < 0
+ })
+ content := ""
+ for idx := range order {
+ content += m[order[idx]]
+ }
+ buf := make([]byte, len(content)+len(cmd.Body))
+ copy(buf, []byte(content))
+ copy(buf[len(content):], cmd.Body)
+
+ cmd.ExtFields[signature] = calculateSignature(buf, []byte(credentials.SecretKey))
+ cmd.ExtFields[accessKey] = credentials.AccessKey
+
+ // The SecurityToken value is unnecessary, user can choose this one.
+ if credentials.SecurityToken != "" {
+ cmd.ExtFields[securityToken] = credentials.SecurityToken
+ }
+ err := next(ctx, req, reply)
+ return err
+ }
+}
+
+func calculateSignature(data, sk []byte) string {
+ mac := hmac.New(func() hash.Hash {
+ return sha1.New()
+ }, sk)
+ mac.Write(data)
+ return base64.StdEncoding.EncodeToString(mac.Sum(nil))
+}
diff --git a/internal/remote/interceptor_test.go b/internal/remote/interceptor_test.go
new file mode 100644
index 0000000..c2cc6ca
--- /dev/null
+++ b/internal/remote/interceptor_test.go
@@ -0,0 +1,29 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remote
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func Test_CalculateSignature(t *testing.T) {
+ assert.Equal(t, "tAb/54Rwwcq+pbH8Loi7FWX4QSQ=",
+ calculateSignature([]byte("Hello RocketMQ Client ACL Feature"), []byte("adiaushdiaushd")))
+}
diff --git a/internal/remote/mock_remote_client.go b/internal/remote/mock_remote_client.go
new file mode 100644
index 0000000..7d7b41c
--- /dev/null
+++ b/internal/remote/mock_remote_client.go
@@ -0,0 +1,123 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+// Code generated by MockGen. DO NOT EDIT.
+// Source: remote_client.go
+
+// Package remote is a generated GoMock package.
+package remote
+
+import (
+ context "context"
+ reflect "reflect"
+
+ primitive "github.com/apache/rocketmq-client-go/v2/primitive"
+ gomock "github.com/golang/mock/gomock"
+)
+
+// MockRemotingClient is a mock of RemotingClient interface
+type MockRemotingClient struct {
+ ctrl *gomock.Controller
+ recorder *MockRemotingClientMockRecorder
+}
+
+// MockRemotingClientMockRecorder is the mock recorder for MockRemotingClient
+type MockRemotingClientMockRecorder struct {
+ mock *MockRemotingClient
+}
+
+// NewMockRemotingClient creates a new mock instance
+func NewMockRemotingClient(ctrl *gomock.Controller) *MockRemotingClient {
+ mock := &MockRemotingClient{ctrl: ctrl}
+ mock.recorder = &MockRemotingClientMockRecorder{mock}
+ return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use
+func (m *MockRemotingClient) EXPECT() *MockRemotingClientMockRecorder {
+ return m.recorder
+}
+
+// RegisterRequestFunc mocks base method
+func (m *MockRemotingClient) RegisterRequestFunc(code int16, f ClientRequestFunc) {
+ m.ctrl.Call(m, "RegisterRequestFunc", code, f)
+}
+
+// RegisterRequestFunc indicates an expected call of RegisterRequestFunc
+func (mr *MockRemotingClientMockRecorder) RegisterRequestFunc(code, f interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterRequestFunc", reflect.TypeOf((*MockRemotingClient)(nil).RegisterRequestFunc), code, f)
+}
+
+// RegisterInterceptor mocks base method
+func (m *MockRemotingClient) RegisterInterceptor(interceptors ...primitive.Interceptor) {
+ varargs := []interface{}{}
+ for _, a := range interceptors {
+ varargs = append(varargs, a)
+ }
+ m.ctrl.Call(m, "RegisterInterceptor", varargs...)
+}
+
+// RegisterInterceptor indicates an expected call of RegisterInterceptor
+func (mr *MockRemotingClientMockRecorder) RegisterInterceptor(interceptors ...interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterInterceptor", reflect.TypeOf((*MockRemotingClient)(nil).RegisterInterceptor), interceptors...)
+}
+
+// InvokeSync mocks base method
+func (m *MockRemotingClient) InvokeSync(ctx context.Context, addr string, request *RemotingCommand) (*RemotingCommand, error) {
+ ret := m.ctrl.Call(m, "InvokeSync", ctx, addr, request)
+ ret0, _ := ret[0].(*RemotingCommand)
+ ret1, _ := ret[1].(error)
+ return ret0, ret1
+}
+
+// InvokeSync indicates an expected call of InvokeSync
+func (mr *MockRemotingClientMockRecorder) InvokeSync(ctx, addr, request interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeSync", reflect.TypeOf((*MockRemotingClient)(nil).InvokeSync), ctx, addr, request)
+}
+
+// InvokeAsync mocks base method
+func (m *MockRemotingClient) InvokeAsync(ctx context.Context, addr string, request *RemotingCommand, callback func(*ResponseFuture)) error {
+ ret := m.ctrl.Call(m, "InvokeAsync", ctx, addr, request, callback)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// InvokeAsync indicates an expected call of InvokeAsync
+func (mr *MockRemotingClientMockRecorder) InvokeAsync(ctx, addr, request, callback interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeAsync", reflect.TypeOf((*MockRemotingClient)(nil).InvokeAsync), ctx, addr, request, callback)
+}
+
+// InvokeOneWay mocks base method
+func (m *MockRemotingClient) InvokeOneWay(ctx context.Context, addr string, request *RemotingCommand) error {
+ ret := m.ctrl.Call(m, "InvokeOneWay", ctx, addr, request)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// InvokeOneWay indicates an expected call of InvokeOneWay
+func (mr *MockRemotingClientMockRecorder) InvokeOneWay(ctx, addr, request interface{}) *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InvokeOneWay", reflect.TypeOf((*MockRemotingClient)(nil).InvokeOneWay), ctx, addr, request)
+}
+
+// ShutDown mocks base method
+func (m *MockRemotingClient) ShutDown() {
+ m.ctrl.Call(m, "ShutDown")
+}
+
+// ShutDown indicates an expected call of ShutDown
+func (mr *MockRemotingClientMockRecorder) ShutDown() *gomock.Call {
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ShutDown", reflect.TypeOf((*MockRemotingClient)(nil).ShutDown))
+}
diff --git a/internal/remote/remote_client.go b/internal/remote/remote_client.go
new file mode 100644
index 0000000..7b5b1ea
--- /dev/null
+++ b/internal/remote/remote_client.go
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package remote
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/binary"
+ "io"
+ "net"
+ "sync"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+type ClientRequestFunc func(*RemotingCommand, net.Addr) *RemotingCommand
+
+type TcpOption struct {
+ // TODO
+}
+
+//go:generate mockgen -source remote_client.go -destination mock_remote_client.go -self_package github.com/apache/rocketmq-client-go/v2/internal/remote --package remote RemotingClient
+type RemotingClient interface {
+ RegisterRequestFunc(code int16, f ClientRequestFunc)
+ RegisterInterceptor(interceptors ...primitive.Interceptor)
+ InvokeSync(ctx context.Context, addr string, request *RemotingCommand) (*RemotingCommand, error)
+ InvokeAsync(ctx context.Context, addr string, request *RemotingCommand, callback func(*ResponseFuture)) error
+ InvokeOneWay(ctx context.Context, addr string, request *RemotingCommand) error
+ ShutDown()
+}
+
+var _ RemotingClient = &remotingClient{}
+
+type remotingClient struct {
+ responseTable sync.Map
+ connectionTable sync.Map
+ option TcpOption
+ processors map[int16]ClientRequestFunc
+ connectionLocker sync.Mutex
+ interceptor primitive.Interceptor
+}
+
+func NewRemotingClient() *remotingClient {
+ return &remotingClient{
+ processors: make(map[int16]ClientRequestFunc),
+ }
+}
+
+func (c *remotingClient) RegisterRequestFunc(code int16, f ClientRequestFunc) {
+ c.processors[code] = f
+}
+
+// TODO: merge sync and async model. sync should run on async model by blocking on chan
+func (c *remotingClient) InvokeSync(ctx context.Context, addr string, request *RemotingCommand) (*RemotingCommand, error) {
+ conn, err := c.connect(ctx, addr)
+ if err != nil {
+ return nil, err
+ }
+ resp := NewResponseFuture(ctx, request.Opaque, nil)
+ c.responseTable.Store(resp.Opaque, resp)
+ defer c.responseTable.Delete(request.Opaque)
+ err = c.sendRequest(conn, request)
+ if err != nil {
+ return nil, err
+ }
+ return resp.waitResponse()
+}
+
+// InvokeAsync send request without blocking, just return immediately.
+func (c *remotingClient) InvokeAsync(ctx context.Context, addr string, request *RemotingCommand, callback func(*ResponseFuture)) error {
+ conn, err := c.connect(ctx, addr)
+ if err != nil {
+ return err
+ }
+ resp := NewResponseFuture(ctx, request.Opaque, callback)
+ c.responseTable.Store(resp.Opaque, resp)
+ err = c.sendRequest(conn, request)
+ if err != nil {
+ return err
+ }
+ go primitive.WithRecover(func() {
+ c.receiveAsync(resp)
+ })
+ return nil
+}
+
+func (c *remotingClient) receiveAsync(f *ResponseFuture) {
+ _, err := f.waitResponse()
+ if err != nil {
+ f.executeInvokeCallback()
+ }
+}
+
+func (c *remotingClient) InvokeOneWay(ctx context.Context, addr string, request *RemotingCommand) error {
+ conn, err := c.connect(ctx, addr)
+ if err != nil {
+ return err
+ }
+ return c.sendRequest(conn, request)
+}
+
+func (c *remotingClient) connect(ctx context.Context, addr string) (*tcpConnWrapper, error) {
+ //it needs additional locker.
+ c.connectionLocker.Lock()
+ defer c.connectionLocker.Unlock()
+ conn, ok := c.connectionTable.Load(addr)
+ if ok {
+ return conn.(*tcpConnWrapper), nil
+ }
+ tcpConn, err := initConn(ctx, addr)
+ if err != nil {
+ return nil, err
+ }
+ c.connectionTable.Store(addr, tcpConn)
+ go primitive.WithRecover(func() {
+ c.receiveResponse(tcpConn)
+ })
+ return tcpConn, nil
+}
+
+func (c *remotingClient) receiveResponse(r *tcpConnWrapper) {
+ var err error
+ header := primitive.GetHeader()
+ defer primitive.BackHeader(header)
+ for {
+ if err != nil {
+ // conn has been closed actively
+ if r.isClosed(err) {
+ return
+ }
+ if err != io.EOF {
+ rlog.Error("conn error, close connection", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ c.closeConnection(r)
+ r.destroy()
+ break
+ }
+
+ _, err = io.ReadFull(r, header)
+ if err != nil {
+ continue
+ }
+
+ var length int32
+ err = binary.Read(bytes.NewReader(header), binary.BigEndian, &length)
+ if err != nil {
+ continue
+ }
+
+ buf := make([]byte, length)
+
+ _, err = io.ReadFull(r, buf)
+ if err != nil {
+ continue
+ }
+
+ cmd, err := decode(buf)
+ if err != nil {
+ rlog.Error("decode RemotingCommand error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ continue
+ }
+ c.processCMD(cmd, r)
+ }
+}
+
+func (c *remotingClient) processCMD(cmd *RemotingCommand, r *tcpConnWrapper) {
+ if cmd.isResponseType() {
+ resp, exist := c.responseTable.Load(cmd.Opaque)
+ if exist {
+ c.responseTable.Delete(cmd.Opaque)
+ responseFuture := resp.(*ResponseFuture)
+ go primitive.WithRecover(func() {
+ responseFuture.ResponseCommand = cmd
+ responseFuture.executeInvokeCallback()
+ if responseFuture.Done != nil {
+ responseFuture.Done <- true
+ }
+ })
+ }
+ } else {
+ f := c.processors[cmd.Code]
+ if f != nil {
+ // single goroutine will be deadlock
+ // TODO: optimize with goroutine pool, https://github.com/apache/rocketmq-client-go/v2/issues/307
+ go primitive.WithRecover(func() {
+ res := f(cmd, r.RemoteAddr())
+ if res != nil {
+ res.Opaque = cmd.Opaque
+ res.Flag |= 1 << 0
+ err := c.sendRequest(r, res)
+ if err != nil {
+ rlog.Warning("send response to broker error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ "responseCode": res.Code,
+ })
+ }
+ }
+ })
+ } else {
+ rlog.Warning("receive broker's requests, but no func to handle", map[string]interface{}{
+ "responseCode": cmd.Code,
+ })
+ }
+ }
+}
+
+func (c *remotingClient) createScanner(r io.Reader) *bufio.Scanner {
+ scanner := bufio.NewScanner(r)
+
+ // max batch size: 32, max message size: 4Mb
+ scanner.Buffer(make([]byte, 1024*1024), 128*1024*1024)
+ scanner.Split(func(data []byte, atEOF bool) (int, []byte, error) {
+ defer func() {
+ if err := recover(); err != nil {
+ rlog.Error("scanner split panic", map[string]interface{}{
+ "panic": err,
+ })
+ }
+ }()
+ if !atEOF {
+ if len(data) >= 4 {
+ var length int32
+ err := binary.Read(bytes.NewReader(data[0:4]), binary.BigEndian, &length)
+ if err != nil {
+ rlog.Error("split data error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ return 0, nil, err
+ }
+
+ if int(length)+4 <= len(data) {
+ return int(length) + 4, data[4 : length+4], nil
+ }
+ }
+ }
+ return 0, nil, nil
+ })
+ return scanner
+}
+
+func (c *remotingClient) sendRequest(conn *tcpConnWrapper, request *RemotingCommand) error {
+ var err error
+ if c.interceptor != nil {
+ err = c.interceptor(context.Background(), request, nil, func(ctx context.Context, req, reply interface{}) error {
+ return c.doRequest(conn, request)
+ })
+ } else {
+ err = c.doRequest(conn, request)
+ }
+ return err
+}
+
+func (c *remotingClient) doRequest(conn *tcpConnWrapper, request *RemotingCommand) error {
+ content, err := encode(request)
+ if err != nil {
+ return err
+ }
+ _, err = conn.Write(content)
+ if err != nil {
+ c.closeConnection(conn)
+ return err
+ }
+ return nil
+}
+
+func (c *remotingClient) closeConnection(toCloseConn *tcpConnWrapper) {
+ c.connectionTable.Range(func(key, value interface{}) bool {
+ if value == toCloseConn {
+ c.connectionTable.Delete(key)
+ return false
+ } else {
+ return true
+ }
+ })
+}
+
+func (c *remotingClient) ShutDown() {
+ c.responseTable.Range(func(key, value interface{}) bool {
+ c.responseTable.Delete(key)
+ return true
+ })
+ c.connectionTable.Range(func(key, value interface{}) bool {
+ conn := value.(*tcpConnWrapper)
+ err := conn.destroy()
+ if err != nil {
+ rlog.Warning("close remoting conn error", map[string]interface{}{
+ "remote": conn.RemoteAddr(),
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ return true
+ })
+}
+
+func (c *remotingClient) RegisterInterceptor(interceptors ...primitive.Interceptor) {
+ c.interceptor = primitive.ChainInterceptors(interceptors...)
+}
diff --git a/internal/remote/remote_client_test.go b/internal/remote/remote_client_test.go
new file mode 100644
index 0000000..5b69fd6
--- /dev/null
+++ b/internal/remote/remote_client_test.go
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package remote
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "math/rand"
+ "net"
+ "reflect"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestNewResponseFuture(t *testing.T) {
+ future := NewResponseFuture(context.Background(), 10, nil)
+ if future.Opaque != 10 {
+ t.Errorf("wrong ResponseFuture's opaque. want=%d, got=%d", 10, future.Opaque)
+ }
+ if future.Err != nil {
+ t.Errorf("wrong RespnseFuture's Err. want=<nil>, got=%v", future.Err)
+ }
+ if future.callback != nil {
+ t.Errorf("wrong ResponseFuture's callback. want=<nil>, got!=<nil>")
+ }
+ if future.Done == nil {
+ t.Errorf("wrong ResponseFuture's done. want=<channel>, got=<nil>")
+ }
+}
+
+func TestResponseFutureTimeout(t *testing.T) {
+ callback := func(r *ResponseFuture) {
+ if r.ResponseCommand.Remark == "" {
+ r.ResponseCommand.Remark = "Hello RocketMQ."
+ } else {
+ r.ResponseCommand.Remark = r.ResponseCommand.Remark + "Go Client"
+ }
+ }
+ future := NewResponseFuture(context.Background(), 10, callback)
+ future.ResponseCommand = NewRemotingCommand(200,
+ nil, nil)
+
+ var wg sync.WaitGroup
+ wg.Add(10)
+ for i := 0; i < 10; i++ {
+ go func() {
+ future.executeInvokeCallback()
+ wg.Done()
+ }()
+ }
+ wg.Wait()
+ if future.ResponseCommand.Remark != "Hello RocketMQ." {
+ t.Errorf("wrong ResponseFuture.ResponseCommand.Remark. want=%s, got=%s",
+ "Hello RocketMQ.", future.ResponseCommand.Remark)
+ }
+
+}
+
+func TestResponseFutureWaitResponse(t *testing.T) {
+ ctx, _ := context.WithTimeout(context.Background(), time.Duration(1000))
+ future := NewResponseFuture(ctx, 10, nil)
+ if _, err := future.waitResponse(); err != utils.ErrRequestTimeout {
+ t.Errorf("wrong ResponseFuture waitResponse. want=%v, got=%v",
+ utils.ErrRequestTimeout, err)
+ }
+ future = NewResponseFuture(context.Background(), 10, nil)
+ responseError := errors.New("response error")
+ go func() {
+ time.Sleep(100 * time.Millisecond)
+ future.Err = responseError
+ future.Done <- true
+ }()
+ if _, err := future.waitResponse(); err != responseError {
+ t.Errorf("wrong ResponseFuture waitResponse. want=%v. got=%v",
+ responseError, err)
+ }
+ future = NewResponseFuture(context.Background(), 10, nil)
+ responseRemotingCommand := NewRemotingCommand(202, nil, nil)
+ go func() {
+ time.Sleep(100 * time.Millisecond)
+ future.ResponseCommand = responseRemotingCommand
+ future.Done <- true
+ }()
+ if r, err := future.waitResponse(); err != nil {
+ t.Errorf("wrong ResponseFuture waitResponse error: %v", err)
+ } else {
+ if r != responseRemotingCommand {
+ t.Errorf("wrong ResponseFuture waitResposne result. want=%v, got=%v",
+ responseRemotingCommand, r)
+ }
+ }
+}
+
+func TestCreateScanner(t *testing.T) {
+ r := randomNewRemotingCommand()
+ content, err := encode(r)
+ if err != nil {
+ t.Fatalf("failed to encode RemotingCommand. %s", err)
+ }
+ client := NewRemotingClient()
+ reader := bytes.NewReader(content)
+ scanner := client.createScanner(reader)
+ for scanner.Scan() {
+ rcr, err := decode(scanner.Bytes())
+ if err != nil {
+ t.Fatalf("failedd to decode RemotingCommand from scanner")
+ }
+ if r.Code != rcr.Code {
+ t.Fatalf("wrong Code. want=%d, got=%d", r.Code, rcr.Code)
+ }
+ if r.Version != rcr.Version {
+ t.Fatalf("wrong Version. want=%d, got=%d", r.Version, rcr.Version)
+ }
+ if r.Opaque != rcr.Opaque {
+ t.Fatalf("wrong opaque. want=%d, got=%d", r.Opaque, rcr.Opaque)
+ }
+ if r.Flag != rcr.Flag {
+ t.Fatalf("wrong flag. want=%d, got=%d", r.Opaque, rcr.Opaque)
+ }
+ if !reflect.DeepEqual(r.ExtFields, rcr.ExtFields) {
+ t.Fatalf("wrong extFields. want=%v, got=%v", r.ExtFields, rcr.ExtFields)
+ }
+ }
+}
+
+func TestInvokeSync(t *testing.T) {
+ addr := ":3004"
+
+ clientSendRemtingCommand := NewRemotingCommand(10, nil, []byte("Hello RocketMQ"))
+ serverSendRemotingCommand := NewRemotingCommand(20, nil, []byte("Welcome native"))
+ serverSendRemotingCommand.Opaque = clientSendRemtingCommand.Opaque
+ serverSendRemotingCommand.Flag = ResponseType
+ var wg sync.WaitGroup
+ wg.Add(1)
+ client := NewRemotingClient()
+
+ var clientSend sync.WaitGroup // blocking client send message until the server listen success.
+ clientSend.Add(1)
+
+ go func() {
+ clientSend.Wait()
+ receiveCommand, err := client.InvokeSync(context.Background(), addr,
+ clientSendRemtingCommand)
+ if err != nil {
+ t.Fatalf("failed to invoke synchronous. %s", err)
+ } else {
+ assert.Equal(t, len(receiveCommand.ExtFields), 0)
+ assert.Equal(t, len(serverSendRemotingCommand.ExtFields), 0)
+ // in order to avoid the difference of ExtFields between the receiveCommand and serverSendRemotingCommand
+ // the ExtFields in receiveCommand is map[string]string(nil), but serverSendRemotingCommand is map[string]string{}
+ receiveCommand.ExtFields = nil
+ serverSendRemotingCommand.ExtFields = nil
+ assert.Equal(t, receiveCommand, serverSendRemotingCommand, "remotingCommand prased in client is different from server.")
+ }
+ wg.Done()
+ }()
+
+ l, err := net.Listen("tcp", addr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ clientSend.Done()
+ for {
+ conn, err := l.Accept()
+ if err != nil {
+ return
+ }
+ defer conn.Close()
+ scanner := client.createScanner(conn)
+ for scanner.Scan() {
+ receivedRemotingCommand, err := decode(scanner.Bytes())
+ if err != nil {
+ t.Errorf("failed to decode RemotingCommnad. %s", err)
+ }
+ if clientSendRemtingCommand.Code != receivedRemotingCommand.Code {
+ t.Errorf("wrong code. want=%d, got=%d", receivedRemotingCommand.Code,
+ clientSendRemtingCommand.Code)
+ }
+ body, err := encode(serverSendRemotingCommand)
+ if err != nil {
+ t.Fatalf("failed to encode RemotingCommand")
+ }
+ _, err = conn.Write(body)
+ if err != nil {
+ t.Fatalf("failed to write body to conneciton.")
+ }
+ goto done
+ }
+ }
+done:
+ wg.Wait()
+}
+
+func TestInvokeAsync(t *testing.T) {
+ addr := ":3006"
+ var wg sync.WaitGroup
+ cnt := 50
+ wg.Add(cnt)
+ client := NewRemotingClient()
+ for i := 0; i < cnt; i++ {
+ go func(index int) {
+ time.Sleep(time.Duration(rand.Intn(100)) * time.Millisecond)
+ t.Logf("[Send: %d] asychronous message", index)
+ sendRemotingCommand := randomNewRemotingCommand()
+ err := client.InvokeAsync(context.Background(), addr, sendRemotingCommand, func(r *ResponseFuture) {
+ t.Logf("[Receive: %d] asychronous message response", index)
+ if string(sendRemotingCommand.Body) != string(r.ResponseCommand.Body) {
+ t.Errorf("wrong response message. want=%s, got=%s", string(sendRemotingCommand.Body),
+ string(r.ResponseCommand.Body))
+ }
+ wg.Done()
+ })
+ if err != nil {
+ t.Errorf("failed to invokeAsync. %s", err)
+ }
+
+ }(i)
+ }
+ l, err := net.Listen("tcp", addr)
+ if err != nil {
+ t.Fatalf("failed to create tcp network. %s", err)
+ }
+ defer l.Close()
+ count := 0
+ for {
+ conn, err := l.Accept()
+ if err != nil {
+ t.Fatalf("failed to create connection. %s", err)
+ }
+ defer conn.Close()
+ scanner := client.createScanner(conn)
+ for scanner.Scan() {
+ t.Log("receive request")
+ r, err := decode(scanner.Bytes())
+ if err != nil {
+ t.Errorf("failed to decode RemotingCommand %s", err)
+ }
+ r.markResponseType()
+ body, _ := encode(r)
+ _, err = conn.Write(body)
+ if err != nil {
+ t.Fatalf("failed to send response %s", err)
+ }
+ count++
+ if count >= cnt {
+ goto done
+ }
+ }
+ }
+done:
+
+ wg.Wait()
+}
+
+func TestInvokeAsyncTimeout(t *testing.T) {
+ addr := ":3002"
+
+ clientSendRemtingCommand := NewRemotingCommand(10, nil, []byte("Hello RocketMQ"))
+ serverSendRemotingCommand := NewRemotingCommand(20, nil, []byte("Welcome native"))
+ serverSendRemotingCommand.Opaque = clientSendRemtingCommand.Opaque
+ serverSendRemotingCommand.Flag = ResponseType
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+ client := NewRemotingClient()
+
+ var clientSend sync.WaitGroup // blocking client send message until the server listen success.
+ clientSend.Add(1)
+ go func() {
+ clientSend.Wait()
+ ctx, _ := context.WithTimeout(context.Background(), time.Duration(10*time.Second))
+ err := client.InvokeAsync(ctx, addr, clientSendRemtingCommand,
+ func(r *ResponseFuture) {
+ assert.NotNil(t, r.Err)
+ assert.Equal(t, utils.ErrRequestTimeout, r.Err)
+ wg.Done()
+ })
+ assert.Nil(t, err, "failed to invokeSync.")
+ }()
+
+ l, err := net.Listen("tcp", addr)
+ assert.Nil(t, err)
+ defer l.Close()
+ clientSend.Done()
+
+ for {
+ conn, err := l.Accept()
+ assert.Nil(t, err)
+ defer conn.Close()
+
+ scanner := client.createScanner(conn)
+ for scanner.Scan() {
+ t.Logf("receive request.")
+ _, err := decode(scanner.Bytes())
+ assert.Nil(t, err, "failed to decode RemotingCommnad.")
+
+ time.Sleep(5 * time.Second) // force client timeout
+ goto done
+ }
+ }
+done:
+ wg.Wait()
+}
+
+func TestInvokeOneWay(t *testing.T) {
+ addr := ":3008"
+ clientSendRemtingCommand := NewRemotingCommand(10, nil, []byte("Hello RocketMQ"))
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+ client := NewRemotingClient()
+
+ var clientSend sync.WaitGroup // blocking client send message until the server listen success.
+ clientSend.Add(1)
+ go func() {
+ clientSend.Wait()
+ err := client.InvokeOneWay(context.Background(), addr, clientSendRemtingCommand)
+ if err != nil {
+ t.Fatalf("failed to invoke synchronous. %s", err)
+ }
+ wg.Done()
+ }()
+
+ l, err := net.Listen("tcp", addr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ clientSend.Done()
+ for {
+ conn, err := l.Accept()
+ if err != nil {
+ return
+ }
+ defer conn.Close()
+ scanner := client.createScanner(conn)
+ for scanner.Scan() {
+ receivedRemotingCommand, err := decode(scanner.Bytes())
+ if err != nil {
+ t.Errorf("failed to decode RemotingCommnad. %s", err)
+ }
+ if clientSendRemtingCommand.Code != receivedRemotingCommand.Code {
+ t.Errorf("wrong code. want=%d, got=%d", receivedRemotingCommand.Code,
+ clientSendRemtingCommand.Code)
+ }
+ goto done
+ }
+ }
+done:
+ wg.Wait()
+}
diff --git a/examples/main.go b/internal/remote/rpchook.go
similarity index 84%
rename from examples/main.go
rename to internal/remote/rpchook.go
index 1e325e9..a95391d 100644
--- a/examples/main.go
+++ b/internal/remote/rpchook.go
@@ -15,15 +15,9 @@
* limitations under the License.
*/
-package main
+package remote
-func main() {
- //run producer
- main0()
- //run consumer
- main1()
- //run orderly producer
- main2()
- //run orderly consumer
- main3()
+type RPCHook interface {
+ DoBeforeRequest(string, *RemotingCommand)
+ DoAfterResponse(string, *RemotingCommand)
}
diff --git a/internal/remote/tcp_conn.go b/internal/remote/tcp_conn.go
new file mode 100644
index 0000000..dda7dcf
--- /dev/null
+++ b/internal/remote/tcp_conn.go
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package remote
+
+import (
+ "context"
+ "net"
+
+ "go.uber.org/atomic"
+)
+
+// TODO: Adding TCP Connections Pool, https://github.com/apache/rocketmq-client-go/v2/issues/298
+type tcpConnWrapper struct {
+ net.Conn
+ closed atomic.Bool
+}
+
+func initConn(ctx context.Context, addr string) (*tcpConnWrapper, error) {
+ var d net.Dialer
+ conn, err := d.DialContext(ctx, "tcp", addr)
+ if err != nil {
+ return nil, err
+ }
+ return &tcpConnWrapper{
+ Conn: conn,
+ }, nil
+}
+
+func (wrapper *tcpConnWrapper) destroy() error {
+ wrapper.closed.Swap(true)
+ return wrapper.Conn.Close()
+}
+
+func (wrapper *tcpConnWrapper) isClosed(err error) bool {
+ if !wrapper.closed.Load() {
+ return false
+ }
+
+ opErr, ok := err.(*net.OpError)
+ if !ok {
+ return false
+ }
+
+ return opErr.Err.Error() == "use of closed network connection"
+}
diff --git a/internal/request.go b/internal/request.go
new file mode 100644
index 0000000..5438790
--- /dev/null
+++ b/internal/request.go
@@ -0,0 +1,320 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "fmt"
+ "strconv"
+ "time"
+)
+
+const (
+ ReqSendMessage = int16(10)
+ ReqPullMessage = int16(11)
+ ReqQueryConsumerOffset = int16(14)
+ ReqUpdateConsumerOffset = int16(15)
+ ReqSearchOffsetByTimestamp = int16(29)
+ ReqGetMaxOffset = int16(30)
+ ReqHeartBeat = int16(34)
+ ReqConsumerSendMsgBack = int16(36)
+ ReqENDTransaction = int16(37)
+ ReqGetConsumerListByGroup = int16(38)
+ ReqLockBatchMQ = int16(41)
+ ReqUnlockBatchMQ = int16(42)
+ ReqGetRouteInfoByTopic = int16(105)
+ ReqSendBatchMessage = int16(320)
+ ReqCheckTransactionState = int16(39)
+ ReqNotifyConsumerIdsChanged = int16(40)
+ ReqResetConsuemrOffset = int16(220)
+ ReqGetConsumerRunningInfo = int16(307)
+ ReqConsumeMessageDirectly = int16(309)
+)
+
+type SendMessageRequestHeader struct {
+ ProducerGroup string
+ Topic string
+ QueueId int
+ SysFlag int
+ BornTimestamp int64
+ Flag int32
+ Properties string
+ ReconsumeTimes int
+ UnitMode bool
+ MaxReconsumeTimes int
+ Batch bool
+ DefaultTopic string
+ DefaultTopicQueueNums string
+}
+
+func (request *SendMessageRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["producerGroup"] = request.ProducerGroup
+ maps["topic"] = request.Topic
+ maps["queueId"] = strconv.Itoa(request.QueueId)
+ maps["sysFlag"] = fmt.Sprintf("%d", request.SysFlag)
+ maps["bornTimestamp"] = strconv.FormatInt(request.BornTimestamp, 10)
+ maps["flag"] = fmt.Sprintf("%d", request.Flag)
+ maps["reconsumeTimes"] = strconv.Itoa(request.ReconsumeTimes)
+ maps["unitMode"] = strconv.FormatBool(request.UnitMode)
+ maps["maxReconsumeTimes"] = strconv.Itoa(request.MaxReconsumeTimes)
+ maps["defaultTopic"] = "TBW102"
+ maps["defaultTopicQueueNums"] = "4"
+ maps["batch"] = strconv.FormatBool(request.Batch)
+ maps["properties"] = request.Properties
+
+ return maps
+}
+
+type EndTransactionRequestHeader struct {
+ ProducerGroup string
+ TranStateTableOffset int64
+ CommitLogOffset int64
+ CommitOrRollback int
+ FromTransactionCheck bool
+ MsgID string
+ TransactionId string
+}
+
+type SendMessageRequestV2Header struct {
+ *SendMessageRequestHeader
+}
+
+func (request *SendMessageRequestV2Header) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["a"] = request.ProducerGroup
+ maps["b"] = request.Topic
+ maps["c"] = request.DefaultTopic
+ maps["d"] = request.DefaultTopicQueueNums
+ maps["e"] = strconv.Itoa(request.QueueId)
+ maps["f"] = fmt.Sprintf("%d", request.SysFlag)
+ maps["g"] = strconv.FormatInt(request.BornTimestamp, 10)
+ maps["h"] = fmt.Sprintf("%d", request.Flag)
+ maps["i"] = request.Properties
+ maps["j"] = strconv.Itoa(request.ReconsumeTimes)
+ maps["k"] = strconv.FormatBool(request.UnitMode)
+ maps["l"] = strconv.Itoa(request.MaxReconsumeTimes)
+ maps["m"] = strconv.FormatBool(request.Batch)
+ return maps
+}
+
+func (request *EndTransactionRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["producerGroup"] = request.ProducerGroup
+ maps["tranStateTableOffset"] = strconv.FormatInt(request.TranStateTableOffset, 10)
+ maps["commitLogOffset"] = strconv.Itoa(int(request.CommitLogOffset))
+ maps["commitOrRollback"] = strconv.Itoa(request.CommitOrRollback)
+ maps["fromTransactionCheck"] = strconv.FormatBool(request.FromTransactionCheck)
+ maps["msgId"] = request.MsgID
+ maps["transactionId"] = request.TransactionId
+ return maps
+}
+
+type CheckTransactionStateRequestHeader struct {
+ TranStateTableOffset int64
+ CommitLogOffset int64
+ MsgId string
+ TransactionId string
+ OffsetMsgId string
+}
+
+func (request *CheckTransactionStateRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["tranStateTableOffset"] = strconv.FormatInt(request.TranStateTableOffset, 10)
+ maps["commitLogOffset"] = strconv.FormatInt(request.CommitLogOffset, 10)
+ maps["msgId"] = request.MsgId
+ maps["transactionId"] = request.TransactionId
+ maps["offsetMsgId"] = request.OffsetMsgId
+
+ return maps
+}
+
+func (request *CheckTransactionStateRequestHeader) Decode(properties map[string]string) {
+ if len(properties) == 0 {
+ return
+ }
+ if v, existed := properties["tranStateTableOffset"]; existed {
+ request.TranStateTableOffset, _ = strconv.ParseInt(v, 10, 0)
+ }
+ if v, existed := properties["commitLogOffset"]; existed {
+ request.CommitLogOffset, _ = strconv.ParseInt(v, 10, 0)
+ }
+ if v, existed := properties["msgId"]; existed {
+ request.MsgId = v
+ }
+ if v, existed := properties["transactionId"]; existed {
+ request.MsgId = v
+ }
+ if v, existed := properties["offsetMsgId"]; existed {
+ request.MsgId = v
+ }
+}
+
+type ConsumerSendMsgBackRequestHeader struct {
+ Group string
+ Offset int64
+ DelayLevel int
+ OriginMsgId string
+ OriginTopic string
+ UnitMode bool
+ MaxReconsumeTimes int32
+}
+
+func (request *ConsumerSendMsgBackRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["group"] = request.Group
+ maps["offset"] = strconv.FormatInt(request.Offset, 10)
+ maps["delayLevel"] = strconv.Itoa(request.DelayLevel)
+ maps["originMsgId"] = request.OriginMsgId
+ maps["originTopic"] = request.OriginTopic
+ maps["unitMode"] = strconv.FormatBool(request.UnitMode)
+ maps["maxReconsumeTimes"] = strconv.Itoa(int(request.MaxReconsumeTimes))
+
+ return maps
+}
+
+type PullMessageRequestHeader struct {
+ ConsumerGroup string
+ Topic string
+ QueueId int32
+ QueueOffset int64
+ MaxMsgNums int32
+ SysFlag int32
+ CommitOffset int64
+ SuspendTimeoutMillis time.Duration
+ SubExpression string
+ SubVersion int64
+ ExpressionType string
+}
+
+func (request *PullMessageRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["consumerGroup"] = request.ConsumerGroup
+ maps["topic"] = request.Topic
+ maps["queueId"] = fmt.Sprintf("%d", request.QueueId)
+ maps["queueOffset"] = fmt.Sprintf("%d", request.QueueOffset)
+ maps["maxMsgNums"] = fmt.Sprintf("%d", request.MaxMsgNums)
+ maps["sysFlag"] = fmt.Sprintf("%d", request.SysFlag)
+ maps["commitOffset"] = fmt.Sprintf("%d", request.CommitOffset)
+ maps["suspendTimeoutMillis"] = fmt.Sprintf("%d", request.SuspendTimeoutMillis/time.Millisecond)
+ maps["subscription"] = request.SubExpression
+ maps["subVersion"] = fmt.Sprintf("%d", request.SubVersion)
+ maps["expressionType"] = request.ExpressionType
+
+ return maps
+}
+
+type GetConsumerListRequestHeader struct {
+ ConsumerGroup string `json:"consumerGroup"`
+}
+
+func (request *GetConsumerListRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["consumerGroup"] = request.ConsumerGroup
+ return maps
+}
+
+type GetMaxOffsetRequestHeader struct {
+ Topic string
+ QueueId int
+}
+
+func (request *GetMaxOffsetRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["topic"] = request.Topic
+ maps["queueId"] = strconv.Itoa(request.QueueId)
+ return maps
+}
+
+type QueryConsumerOffsetRequestHeader struct {
+ ConsumerGroup string
+ Topic string
+ QueueId int
+}
+
+func (request *QueryConsumerOffsetRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["consumerGroup"] = request.ConsumerGroup
+ maps["topic"] = request.Topic
+ maps["queueId"] = strconv.Itoa(request.QueueId)
+ return maps
+}
+
+type SearchOffsetRequestHeader struct {
+ Topic string
+ QueueId int
+ Timestamp int64
+}
+
+func (request *SearchOffsetRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["topic"] = request.Topic
+ maps["queueId"] = strconv.Itoa(request.QueueId)
+ maps["timestamp"] = strconv.FormatInt(request.Timestamp, 10)
+ return maps
+}
+
+type UpdateConsumerOffsetRequestHeader struct {
+ ConsumerGroup string
+ Topic string
+ QueueId int
+ CommitOffset int64
+}
+
+func (request *UpdateConsumerOffsetRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["consumerGroup"] = request.ConsumerGroup
+ maps["topic"] = request.Topic
+ maps["queueId"] = strconv.Itoa(request.QueueId)
+ maps["commitOffset"] = strconv.FormatInt(request.CommitOffset, 10)
+ return maps
+}
+
+type GetRouteInfoRequestHeader struct {
+ Topic string
+}
+
+func (request *GetRouteInfoRequestHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["topic"] = request.Topic
+ return maps
+}
+
+type GetConsumerRunningInfoHeader struct {
+ consumerGroup string
+ clientID string
+}
+
+func (request *GetConsumerRunningInfoHeader) Encode() map[string]string {
+ maps := make(map[string]string)
+ maps["consumerGroup"] = request.consumerGroup
+ maps["clientId"] = request.clientID
+ return maps
+}
+
+func (request *GetConsumerRunningInfoHeader) Decode(properties map[string]string) {
+ if len(properties) == 0 {
+ return
+ }
+ if v, existed := properties["consumerGroup"]; existed {
+ request.consumerGroup = v
+ }
+
+ if v, existed := properties["clientId"]; existed {
+ request.clientID = v
+ }
+}
diff --git a/internal/response.go b/internal/response.go
new file mode 100644
index 0000000..ae75b9c
--- /dev/null
+++ b/internal/response.go
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+const (
+ ResSuccess = int16(0)
+ ResError = int16(1)
+ ResFlushDiskTimeout = int16(10)
+ ResSlaveNotAvailable = int16(11)
+ ResFlushSlaveTimeout = int16(12)
+ ResTopicNotExist = int16(17)
+ ResPullNotFound = int16(19)
+ ResPullRetryImmediately = int16(20)
+ ResPullOffsetMoved = int16(21)
+)
+
+type SendMessageResponse struct {
+ MsgId string
+ QueueId int32
+ QueueOffset int64
+ TransactionId string
+ MsgRegion string
+}
+
+func (response *SendMessageResponse) Decode(properties map[string]string) {
+
+}
+
+type PullMessageResponse struct {
+ SuggestWhichBrokerId int64
+ NextBeginOffset int64
+ MinOffset int64
+ MaxOffset int64
+}
diff --git a/internal/route.go b/internal/route.go
new file mode 100644
index 0000000..09b6e53
--- /dev/null
+++ b/internal/route.go
@@ -0,0 +1,621 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "context"
+ "errors"
+ "math/rand"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ jsoniter "github.com/json-iterator/go"
+ "github.com/tidwall/gjson"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+const (
+ EnvNameServerAddr = "NAMESRV_ADDR"
+
+ requestTimeout = 3 * time.Second
+ defaultTopic = "TBW102"
+ defaultQueueNums = 4
+ MasterId = int64(0)
+)
+
+var (
+ ErrTopicNotExist = errors.New("topic not exist")
+)
+
+func (s *namesrvs) cleanOfflineBroker() {
+ // TODO optimize
+ s.lockNamesrv.Lock()
+ s.brokerAddressesMap.Range(func(key, value interface{}) bool {
+ brokerName := key.(string)
+ bd := value.(*BrokerData)
+ for k, v := range bd.BrokerAddresses {
+ isBrokerAddrExistInTopicRoute := false
+ s.routeDataMap.Range(func(key, value interface{}) bool {
+ trd := value.(*TopicRouteData)
+ for idx := range trd.BrokerDataList {
+ for _, v1 := range trd.BrokerDataList[idx].BrokerAddresses {
+ if v1 == v {
+ isBrokerAddrExistInTopicRoute = true
+ return false
+ }
+ }
+ }
+ return true
+ })
+ if !isBrokerAddrExistInTopicRoute {
+ delete(bd.BrokerAddresses, k)
+ rlog.Info("the broker: [name=%s, ID=%d, addr=%s,] is offline, remove it", map[string]interface{}{
+ "brokerName": brokerName,
+ "brokerID": k,
+ "brokerAddr": v,
+ })
+ }
+ }
+ if len(bd.BrokerAddresses) == 0 {
+ s.brokerAddressesMap.Delete(brokerName)
+ rlog.Info("the broker name's host is offline, remove it", map[string]interface{}{
+ "brokerName": brokerName,
+ })
+ }
+ return true
+ })
+ s.lockNamesrv.Unlock()
+}
+
+// key is topic, value is TopicPublishInfo
+type TopicPublishInfo struct {
+ OrderTopic bool
+ HaveTopicRouterInfo bool
+ MqList []*primitive.MessageQueue
+ RouteData *TopicRouteData
+ TopicQueueIndex int32
+}
+
+func (info *TopicPublishInfo) isOK() (bIsTopicOk bool) {
+ return len(info.MqList) > 0
+}
+
+func (info *TopicPublishInfo) fetchQueueIndex() int {
+ length := len(info.MqList)
+ if length <= 0 {
+ return -1
+ }
+ qIndex := atomic.AddInt32(&info.TopicQueueIndex, 1)
+ return int(qIndex) % length
+}
+
+func (s *namesrvs) UpdateTopicRouteInfo(topic string) (*TopicRouteData, bool, error) {
+ return s.UpdateTopicRouteInfoWithDefault(topic, "", 0)
+}
+
+func (s *namesrvs) UpdateTopicRouteInfoWithDefault(topic string, defaultTopic string, defaultQueueNum int) (*TopicRouteData, bool, error) {
+ s.lockNamesrv.Lock()
+ defer s.lockNamesrv.Unlock()
+
+ var (
+ routeData *TopicRouteData
+ err error
+ )
+
+ t := topic
+ if len(defaultTopic) > 0 {
+ t = defaultTopic
+ }
+ routeData, err = s.queryTopicRouteInfoFromServer(t)
+
+ if err != nil {
+ rlog.Warning("query topic route from server error", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+
+ if routeData == nil {
+ rlog.Warning("queryTopicRouteInfoFromServer return nil", map[string]interface{}{
+ rlog.LogKeyTopic: topic,
+ })
+ return nil, false, err
+ }
+
+ if len(defaultTopic) > 0 {
+ for _, q := range routeData.QueueDataList {
+ if q.ReadQueueNums > defaultQueueNum {
+ q.ReadQueueNums = defaultQueueNum
+ q.WriteQueueNums = defaultQueueNum
+ }
+ }
+ }
+
+ oldRouteData, exist := s.routeDataMap.Load(topic)
+
+ changed := true
+ if exist {
+ changed = s.topicRouteDataIsChange(oldRouteData.(*TopicRouteData), routeData)
+ }
+
+ if changed {
+ s.routeDataMap.Store(topic, routeData)
+ rlog.Info("the topic route info changed", map[string]interface{}{
+ rlog.LogKeyTopic: topic,
+ rlog.LogKeyValueChangedFrom: oldRouteData,
+ rlog.LogKeyValueChangedTo: routeData.String(),
+ })
+ for _, brokerData := range routeData.BrokerDataList {
+ s.brokerAddressesMap.Store(brokerData.BrokerName, brokerData)
+ }
+ }
+
+ return routeData.clone(), changed, nil
+}
+
+func (s *namesrvs) AddBroker(routeData *TopicRouteData) {
+ for _, brokerData := range routeData.BrokerDataList {
+ s.brokerAddressesMap.Store(brokerData.BrokerName, brokerData)
+ }
+}
+
+func (s *namesrvs) FindBrokerAddrByTopic(topic string) string {
+ v, exist := s.routeDataMap.Load(topic)
+ if !exist {
+ return ""
+ }
+ routeData := v.(*TopicRouteData)
+ if len(routeData.BrokerDataList) == 0 {
+ return ""
+ }
+ i := utils.AbsInt(rand.Int())
+ bd := routeData.BrokerDataList[i%len(routeData.BrokerDataList)]
+ addr := bd.BrokerAddresses[MasterId]
+ if addr == "" && len(bd.BrokerAddresses) > 0 {
+ i = i % len(bd.BrokerAddresses)
+ for _, v := range bd.BrokerAddresses {
+ if i <= 0 {
+ addr = v
+ break
+ }
+ i--
+ }
+ }
+ return addr
+}
+
+func (s *namesrvs) FindBrokerAddrByName(brokerName string) string {
+ bd, exist := s.brokerAddressesMap.Load(brokerName)
+
+ if !exist {
+ return ""
+ }
+
+ return bd.(*BrokerData).BrokerAddresses[MasterId]
+}
+
+func (s *namesrvs) FindBrokerAddressInSubscribe(brokerName string, brokerId int64, onlyThisBroker bool) *FindBrokerResult {
+ var (
+ brokerAddr = ""
+ //slave = false
+ //found = false
+ )
+
+ v, exist := s.brokerAddressesMap.Load(brokerName)
+
+ if !exist {
+ return nil
+ }
+ data := v.(*BrokerData)
+ if len(data.BrokerAddresses) == 0 {
+ return nil
+ }
+
+ brokerAddr = data.BrokerAddresses[brokerId]
+ //for k, v := range data.BrokerAddresses {
+ // if v != "" {
+ // found = true
+ // if k != MasterId {
+ // slave = true
+ // }
+ // brokerAddr = v
+ // break
+ // }
+ //}
+
+ var result *FindBrokerResult
+ if brokerAddr != "" {
+ result = &FindBrokerResult{
+ BrokerAddr: brokerAddr,
+ Slave: brokerId != 0,
+ BrokerVersion: s.findBrokerVersion(brokerName, brokerAddr),
+ }
+ }
+
+ return result
+}
+
+func (s *namesrvs) FetchSubscribeMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
+ routeData, err := s.queryTopicRouteInfoFromServer(topic)
+
+ if err != nil {
+ return nil, err
+ }
+
+ mqs := make([]*primitive.MessageQueue, 0)
+
+ for _, qd := range routeData.QueueDataList {
+ if queueIsReadable(qd.Perm) {
+ for i := 0; i < qd.ReadQueueNums; i++ {
+ mqs = append(mqs, &primitive.MessageQueue{Topic: topic, BrokerName: qd.BrokerName, QueueId: i})
+ }
+ }
+ }
+ return mqs, nil
+}
+
+func (s *namesrvs) FetchPublishMessageQueues(topic string) ([]*primitive.MessageQueue, error) {
+ var (
+ err error
+ routeData *TopicRouteData
+ )
+
+ v, exist := s.routeDataMap.Load(topic)
+ if !exist {
+ routeData, err = s.queryTopicRouteInfoFromServer(topic)
+ if err != nil {
+ rlog.Error("queryTopicRouteInfoFromServer failed", map[string]interface{}{
+ rlog.LogKeyTopic: topic,
+ })
+ return nil, err
+ }
+ s.routeDataMap.Store(topic, routeData)
+ s.AddBroker(routeData)
+ } else {
+ routeData = v.(*TopicRouteData)
+ }
+
+ if err != nil {
+ return nil, err
+ }
+ publishInfo := s.routeData2PublishInfo(topic, routeData)
+
+ return publishInfo.MqList, nil
+}
+
+func (s *namesrvs) AddBrokerVersion(brokerName, brokerAddr string, version int32) {
+ s.brokerLock.Lock()
+ defer s.brokerLock.Unlock()
+
+ m, exist := s.brokerVersionMap[brokerName]
+ if !exist {
+ m = make(map[string]int32, 4)
+ s.brokerVersionMap[brokerName] = m
+ }
+ m[brokerAddr] = version
+}
+
+func (s *namesrvs) findBrokerVersion(brokerName, brokerAddr string) int32 {
+ s.brokerLock.RLock()
+ defer s.brokerLock.RUnlock()
+
+ versions, exist := s.brokerVersionMap[brokerName]
+
+ if !exist {
+ return 0
+ }
+
+ return versions[brokerAddr]
+}
+
+func (s *namesrvs) queryTopicRouteInfoFromServer(topic string) (*TopicRouteData, error) {
+ request := &GetRouteInfoRequestHeader{
+ Topic: topic,
+ }
+
+ var (
+ response *remote.RemotingCommand
+ err error
+ )
+
+ //if s.Size() == 0, response will be nil, lead to panic below.
+ if s.Size() == 0 {
+ rlog.Error("namesrv list empty. UpdateNameServerAddress should be called first.", map[string]interface{}{
+ "namesrv": s,
+ "topic": topic,
+ })
+ return nil, primitive.NewRemotingErr("namesrv list empty")
+ }
+
+ for i := 0; i < s.Size(); i++ {
+ rc := remote.NewRemotingCommand(ReqGetRouteInfoByTopic, request, nil)
+ ctx, _ := context.WithTimeout(context.Background(), requestTimeout)
+ response, err = s.nameSrvClient.InvokeSync(ctx, s.getNameServerAddress(), rc)
+
+ if err == nil {
+ break
+ }
+ }
+ if err != nil {
+ rlog.Error("connect to namesrv failed.", map[string]interface{}{
+ "namesrv": s,
+ "topic": topic,
+ })
+ return nil, primitive.NewRemotingErr(err.Error())
+ }
+
+ switch response.Code {
+ case ResSuccess:
+ if response.Body == nil {
+ return nil, primitive.NewMQClientErr(response.Code, response.Remark)
+ }
+ routeData := &TopicRouteData{}
+
+ err = routeData.decode(string(response.Body))
+ if err != nil {
+ rlog.Warning("decode TopicRouteData error: %s", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ "topic": topic,
+ })
+ return nil, err
+ }
+ return routeData, nil
+ case ResTopicNotExist:
+ return nil, ErrTopicNotExist
+ default:
+ return nil, primitive.NewMQClientErr(response.Code, response.Remark)
+ }
+}
+
+func (s *namesrvs) topicRouteDataIsChange(oldData *TopicRouteData, newData *TopicRouteData) bool {
+ if oldData == nil || newData == nil {
+ return true
+ }
+ oldDataCloned := oldData.clone()
+ newDataCloned := newData.clone()
+
+ sort.Slice(oldDataCloned.QueueDataList, func(i, j int) bool {
+ return strings.Compare(oldDataCloned.QueueDataList[i].BrokerName, oldDataCloned.QueueDataList[j].BrokerName) > 0
+ })
+ sort.Slice(oldDataCloned.BrokerDataList, func(i, j int) bool {
+ return strings.Compare(oldDataCloned.BrokerDataList[i].BrokerName, oldDataCloned.BrokerDataList[j].BrokerName) > 0
+ })
+ sort.Slice(newDataCloned.QueueDataList, func(i, j int) bool {
+ return strings.Compare(newDataCloned.QueueDataList[i].BrokerName, newDataCloned.QueueDataList[j].BrokerName) > 0
+ })
+ sort.Slice(newDataCloned.BrokerDataList, func(i, j int) bool {
+ return strings.Compare(newDataCloned.BrokerDataList[i].BrokerName, newDataCloned.BrokerDataList[j].BrokerName) > 0
+ })
+
+ return !oldDataCloned.equals(newDataCloned)
+}
+
+func (s *namesrvs) routeData2PublishInfo(topic string, data *TopicRouteData) *TopicPublishInfo {
+ publishInfo := &TopicPublishInfo{
+ RouteData: data,
+ OrderTopic: false,
+ }
+
+ if data.OrderTopicConf != "" {
+ brokers := strings.Split(data.OrderTopicConf, ";")
+ for _, broker := range brokers {
+ item := strings.Split(broker, ":")
+ nums, _ := strconv.Atoi(item[1])
+ for i := 0; i < nums; i++ {
+ mq := &primitive.MessageQueue{
+ Topic: topic,
+ BrokerName: item[0],
+ QueueId: i,
+ }
+ publishInfo.MqList = append(publishInfo.MqList, mq)
+ }
+ }
+
+ publishInfo.OrderTopic = true
+ return publishInfo
+ }
+
+ qds := data.QueueDataList
+ sort.Slice(qds, func(i, j int) bool {
+ return i-j >= 0
+ })
+
+ for _, qd := range qds {
+ if !queueIsWriteable(qd.Perm) {
+ continue
+ }
+
+ var bData *BrokerData
+ for _, bd := range data.BrokerDataList {
+ if bd.BrokerName == qd.BrokerName {
+ bData = bd
+ break
+ }
+ }
+
+ if bData == nil || bData.BrokerAddresses[MasterId] == "" {
+ continue
+ }
+
+ for i := 0; i < qd.WriteQueueNums; i++ {
+ mq := &primitive.MessageQueue{
+ Topic: topic,
+ BrokerName: qd.BrokerName,
+ QueueId: i,
+ }
+ publishInfo.MqList = append(publishInfo.MqList, mq)
+ }
+ }
+
+ return publishInfo
+}
+
+// TopicRouteData TopicRouteData
+type TopicRouteData struct {
+ OrderTopicConf string
+ QueueDataList []*QueueData `json:"queueDatas"`
+ BrokerDataList []*BrokerData `json:"brokerDatas"`
+}
+
+func (routeData *TopicRouteData) decode(data string) error {
+ res := gjson.Parse(data)
+ err := jsoniter.Unmarshal([]byte(res.Get("queueDatas").String()), &routeData.QueueDataList)
+
+ if err != nil {
+ return err
+ }
+
+ bds := res.Get("brokerDatas").Array()
+ routeData.BrokerDataList = make([]*BrokerData, len(bds))
+ for idx, v := range bds {
+ bd := &BrokerData{
+ BrokerName: v.Get("brokerName").String(),
+ Cluster: v.Get("cluster").String(),
+ BrokerAddresses: make(map[int64]string, 0),
+ }
+ addrs := v.Get("brokerAddrs").String()
+ strs := strings.Split(addrs[1:len(addrs)-1], ",")
+ if strs != nil {
+ for _, str := range strs {
+ i := strings.Index(str, ":")
+ if i < 0 {
+ continue
+ }
+ id, _ := strconv.ParseInt(str[0:i], 10, 64)
+ bd.BrokerAddresses[id] = strings.Replace(str[i+1:], "\"", "", -1)
+ }
+ }
+ routeData.BrokerDataList[idx] = bd
+ }
+ return nil
+}
+
+func (routeData *TopicRouteData) clone() *TopicRouteData {
+ cloned := &TopicRouteData{
+ OrderTopicConf: routeData.OrderTopicConf,
+ QueueDataList: make([]*QueueData, len(routeData.QueueDataList)),
+ BrokerDataList: make([]*BrokerData, len(routeData.BrokerDataList)),
+ }
+
+ for index, value := range routeData.QueueDataList {
+ cloned.QueueDataList[index] = value
+ }
+
+ for index, value := range routeData.BrokerDataList {
+ cloned.BrokerDataList[index] = value
+ }
+
+ return cloned
+}
+
+func (routeData *TopicRouteData) equals(data *TopicRouteData) bool {
+ if len(routeData.BrokerDataList) != len(data.BrokerDataList) {
+ return false
+ }
+ if len(routeData.QueueDataList) != len(data.QueueDataList) {
+ return false
+ }
+
+ for idx := range routeData.BrokerDataList {
+ if !routeData.BrokerDataList[idx].Equals(data.BrokerDataList[idx]) {
+ return false
+ }
+ }
+
+ for idx := range routeData.QueueDataList {
+ if !routeData.QueueDataList[idx].Equals(data.QueueDataList[idx]) {
+ return false
+ }
+ }
+ return true
+}
+
+func (routeData *TopicRouteData) String() string {
+ data, _ := jsoniter.Marshal(routeData)
+ return string(data)
+}
+
+// QueueData QueueData
+type QueueData struct {
+ BrokerName string `json:"brokerName"`
+ ReadQueueNums int `json:"readQueueNums"`
+ WriteQueueNums int `json:"writeQueueNums"`
+ Perm int `json:"perm"`
+ TopicSynFlag int `json:"topicSynFlag"`
+}
+
+func (q *QueueData) Equals(qd *QueueData) bool {
+ if q.BrokerName != qd.BrokerName {
+ return false
+ }
+
+ if q.ReadQueueNums != qd.ReadQueueNums {
+ return false
+ }
+
+ if q.WriteQueueNums != qd.WriteQueueNums {
+ return false
+ }
+
+ if q.Perm != qd.Perm {
+ return false
+ }
+
+ if q.TopicSynFlag != qd.TopicSynFlag {
+ return false
+ }
+
+ return true
+}
+
+// BrokerData BrokerData
+type BrokerData struct {
+ Cluster string `json:"cluster"`
+ BrokerName string `json:"brokerName"`
+ BrokerAddresses map[int64]string `json:"brokerAddrs"`
+ brokerAddressesLock sync.RWMutex
+}
+
+func (b *BrokerData) Equals(bd *BrokerData) bool {
+ if b.Cluster != bd.Cluster {
+ return false
+ }
+
+ if b.BrokerName != bd.BrokerName {
+ return false
+ }
+
+ if len(b.BrokerAddresses) != len(bd.BrokerAddresses) {
+ return false
+ }
+
+ for k, v := range b.BrokerAddresses {
+ if bd.BrokerAddresses[k] != v {
+ return false
+ }
+ }
+
+ return true
+}
diff --git a/internal/route_test.go b/internal/route_test.go
new file mode 100644
index 0000000..c9b65f0
--- /dev/null
+++ b/internal/route_test.go
@@ -0,0 +1,84 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "context"
+ "sync"
+ "testing"
+
+ "github.com/golang/mock/gomock"
+ "github.com/pkg/errors"
+ . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func TestQueryTopicRouteInfoFromServer(t *testing.T) {
+ Convey("marshal of TraceContext", t, func() {
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+
+ remotingCli := remote.NewMockRemotingClient(ctrl)
+
+ addr, err := primitive.NewNamesrvAddr("1.1.1.1:8880", "1.1.1.2:8880", "1.1.1.3:8880")
+ assert.Nil(t, err)
+
+ namesrv, err := NewNamesrv(primitive.NewPassthroughResolver(addr))
+ assert.Nil(t, err)
+ namesrv.nameSrvClient = remotingCli
+
+ Convey("When marshal producer trace data", func() {
+
+ count := 0
+ remotingCli.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(
+ func(ctx context.Context, addr string, request *remote.RemotingCommand) (*remote.RemotingCommand, error) {
+ count++
+ if count < 3 {
+ return nil, errors.New("not existed")
+ }
+ return &remote.RemotingCommand{
+ Code: ResTopicNotExist,
+ }, nil
+ }).Times(3)
+
+ data, err := namesrv.queryTopicRouteInfoFromServer("notexisted")
+ assert.Nil(t, data)
+ assert.Equal(t, ErrTopicNotExist, err)
+ })
+ })
+}
+
+func TestAddBrokerVersion(t *testing.T) {
+ s := &namesrvs{}
+ s.brokerVersionMap = make(map[string]map[string]int32, 0)
+ s.brokerLock = new(sync.RWMutex)
+
+ v := s.findBrokerVersion("b1", "addr1")
+ assert.Equal(t, v, int32(0))
+
+ s.AddBrokerVersion("b1", "addr1", 1)
+ v = s.findBrokerVersion("b1", "addr1")
+ assert.Equal(t, v, int32(1))
+
+ v = s.findBrokerVersion("b1", "addr2")
+ assert.Equal(t, v, int32(0))
+}
diff --git a/internal/trace.go b/internal/trace.go
new file mode 100644
index 0000000..f0c643c
--- /dev/null
+++ b/internal/trace.go
@@ -0,0 +1,502 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "runtime"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+type TraceBean struct {
+ Topic string
+ MsgId string
+ OffsetMsgId string
+ Tags string
+ Keys string
+ StoreHost string
+ ClientHost string
+ StoreTime int64
+ RetryTimes int
+ BodyLength int
+ MsgType primitive.MessageType
+}
+
+type TraceTransferBean struct {
+ transData string
+ // not duplicate
+ transKey []string
+}
+
+type TraceType string
+
+const (
+ Pub TraceType = "Pub"
+ SubBefore TraceType = "SubBefore"
+ SubAfter TraceType = "SubAfter"
+
+ contentSplitter = '\001'
+ fieldSplitter = '\002'
+)
+
+type TraceContext struct {
+ TraceType TraceType
+ TimeStamp int64
+ RegionId string
+ RegionName string
+ GroupName string
+ CostTime int64
+ IsSuccess bool
+ RequestId string
+ ContextCode int
+ TraceBeans []TraceBean
+}
+
+func (ctx *TraceContext) marshal2Bean() *TraceTransferBean {
+ buffer := bytes.NewBufferString("")
+ switch ctx.TraceType {
+ case Pub:
+ bean := ctx.TraceBeans[0]
+ buffer.WriteString(string(ctx.TraceType))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.FormatInt(ctx.TimeStamp, 10))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(ctx.RegionId)
+ buffer.WriteRune(contentSplitter)
+ ss := strings.Split(ctx.GroupName, "%")
+ if len(ss) == 2 {
+ buffer.WriteString(ss[1])
+ } else {
+ buffer.WriteString(ctx.GroupName)
+ }
+
+ buffer.WriteRune(contentSplitter)
+ ssTopic := strings.Split(bean.Topic, "%")
+ if len(ssTopic) == 2 {
+ buffer.WriteString(ssTopic[1])
+ } else {
+ buffer.WriteString(bean.Topic)
+ }
+ //buffer.WriteString(bean.Topic)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(bean.MsgId)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(bean.Tags)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(bean.Keys)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(bean.StoreHost)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.Itoa(bean.BodyLength))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.FormatInt(ctx.CostTime, 10))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.Itoa(int(bean.MsgType)))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(bean.OffsetMsgId)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.FormatBool(ctx.IsSuccess))
+ buffer.WriteRune(fieldSplitter)
+ case SubBefore:
+ for _, bean := range ctx.TraceBeans {
+ buffer.WriteString(string(ctx.TraceType))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.FormatInt(ctx.TimeStamp, 10))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(ctx.RegionId)
+ buffer.WriteRune(contentSplitter)
+ ss := strings.Split(ctx.GroupName, "%")
+ if len(ss) == 2 {
+ buffer.WriteString(ss[1])
+ } else {
+ buffer.WriteString(ctx.GroupName)
+ }
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(ctx.RequestId)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(bean.MsgId)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.Itoa(bean.RetryTimes))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(nullWrap(bean.Keys))
+ buffer.WriteRune(fieldSplitter)
+ }
+ case SubAfter:
+ for _, bean := range ctx.TraceBeans {
+ buffer.WriteString(string(ctx.TraceType))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(ctx.RequestId)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(bean.MsgId)
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.FormatInt(ctx.CostTime, 10))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.FormatBool(ctx.IsSuccess))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(nullWrap(bean.Keys))
+ buffer.WriteRune(contentSplitter)
+ buffer.WriteString(strconv.Itoa(ctx.ContextCode))
+ buffer.WriteRune(fieldSplitter)
+ }
+ }
+ transferBean := new(TraceTransferBean)
+ transferBean.transData = buffer.String()
+ for _, bean := range ctx.TraceBeans {
+ transferBean.transKey = append(transferBean.transKey, bean.MsgId)
+ if len(bean.Keys) > 0 {
+ transferBean.transKey = append(transferBean.transKey, bean.Keys)
+ }
+ }
+ return transferBean
+}
+
+// compatible with java console.
+func nullWrap(s string) string {
+ if len(s) == 0 {
+ return "null"
+ }
+ return s
+}
+
+type traceDispatcherType int
+
+const (
+ RmqSysTraceTopic = "RMQ_SYS_TRACE_TOPIC"
+
+ ProducerType traceDispatcherType = iota
+ ConsumerType
+
+ maxMsgSize = 128000 - 10*1000
+ batchSize = 100
+
+ TraceTopicPrefix = SystemTopicPrefix + "TRACE_DATA_"
+ TraceGroupName = "_INNER_TRACE_PRODUCER"
+)
+
+type TraceDispatcher interface {
+ GetTraceTopicName() string
+
+ Start()
+ Append(ctx TraceContext) bool
+ Close()
+}
+
+type traceDispatcher struct {
+ ctx context.Context
+ cancel context.CancelFunc
+ running bool
+
+ traceTopic string
+ access primitive.AccessChannel
+
+ ticker *time.Ticker
+ input chan TraceContext
+ batchCh chan []*TraceContext
+
+ discardCount int64
+
+ // support deliver trace message to other cluster.
+ namesrvs *namesrvs
+ // round robin index
+ rrindex int32
+ cli RMQClient
+}
+
+func NewTraceDispatcher(traceCfg *primitive.TraceConfig) *traceDispatcher {
+ ctx := context.Background()
+ ctx, cancel := context.WithCancel(ctx)
+
+ t := traceCfg.TraceTopic
+ if len(t) == 0 {
+ t = RmqSysTraceTopic
+ }
+
+ if traceCfg.Access == primitive.Cloud {
+ t = TraceTopicPrefix + traceCfg.TraceTopic
+ }
+
+ srvs, err := NewNamesrv(primitive.NewPassthroughResolver(traceCfg.NamesrvAddrs))
+ if err != nil {
+ panic(errors.Wrap(err, "new Namesrv failed."))
+ }
+ if !traceCfg.Credentials.IsEmpty() {
+ srvs.SetCredentials(traceCfg.Credentials)
+ }
+
+ cliOp := DefaultClientOptions()
+ cliOp.GroupName = traceCfg.GroupName
+ cliOp.NameServerAddrs = traceCfg.NamesrvAddrs
+ cliOp.InstanceName = "INNER_TRACE_CLIENT_DEFAULT"
+ cliOp.RetryTimes = 0
+ cliOp.Namesrv = srvs
+ cliOp.Credentials = traceCfg.Credentials
+ cli := GetOrNewRocketMQClient(cliOp, nil)
+ return &traceDispatcher{
+ ctx: ctx,
+ cancel: cancel,
+
+ traceTopic: t,
+ access: traceCfg.Access,
+ input: make(chan TraceContext, 1024),
+ batchCh: make(chan []*TraceContext, 2048),
+ cli: cli,
+ namesrvs: srvs,
+ }
+}
+
+func (td *traceDispatcher) GetTraceTopicName() string {
+ return td.traceTopic
+}
+
+func (td *traceDispatcher) Start() {
+ td.running = true
+ td.cli.Start()
+ go primitive.WithRecover(func() {
+ td.process()
+ })
+}
+
+func (td *traceDispatcher) Close() {
+ td.running = false
+ td.ticker.Stop()
+ td.cancel()
+}
+
+func (td *traceDispatcher) Append(ctx TraceContext) bool {
+ if !td.running {
+ rlog.Error("traceDispatcher is closed.", nil)
+ return false
+ }
+ select {
+ case td.input <- ctx:
+ return true
+ default:
+ rlog.Warning("buffer full", map[string]interface{}{
+ "discardCount": atomic.AddInt64(&td.discardCount, 1),
+ "TraceContext": ctx,
+ })
+ return false
+ }
+}
+
+// process
+func (td *traceDispatcher) process() {
+ var count int
+ var batch []TraceContext
+ maxWaitDuration := 5 * time.Millisecond
+ maxWaitTime := maxWaitDuration.Nanoseconds()
+ td.ticker = time.NewTicker(maxWaitDuration)
+ lastput := time.Now()
+ for {
+ select {
+ case ctx := <-td.input:
+ count++
+ lastput = time.Now()
+ batch = append(batch, ctx)
+ if count == batchSize {
+ count = 0
+ batchSend := batch
+ go primitive.WithRecover(func() {
+ td.batchCommit(batchSend)
+ })
+ batch = make([]TraceContext, 0)
+ }
+ case <-td.ticker.C:
+ delta := time.Since(lastput).Nanoseconds()
+ if delta > maxWaitTime {
+ count++
+ lastput = time.Now()
+ if len(batch) > 0 {
+ batchSend := batch
+ go primitive.WithRecover(func() {
+ td.batchCommit(batchSend)
+ })
+ batch = make([]TraceContext, 0)
+ }
+ }
+ case <-td.ctx.Done():
+ batchSend := batch
+ go primitive.WithRecover(func() {
+ td.batchCommit(batchSend)
+ })
+ batch = make([]TraceContext, 0)
+
+ now := time.Now().UnixNano() / int64(time.Millisecond)
+ end := now + 500
+ for now < end {
+ now = time.Now().UnixNano() / int64(time.Millisecond)
+ runtime.Gosched()
+ }
+ rlog.Info(fmt.Sprintf("------end trace send %v %v", td.input, td.batchCh), nil)
+ }
+ }
+}
+
+// batchCommit commit slice of TraceContext. convert the ctxs to keyed pair(key is Topic + regionid).
+// flush according key one by one.
+func (td *traceDispatcher) batchCommit(ctxs []TraceContext) {
+ keyedCtxs := make(map[string][]TraceTransferBean)
+ for _, ctx := range ctxs {
+ if len(ctx.TraceBeans) == 0 {
+ return
+ }
+ topic := ctx.TraceBeans[0].Topic
+ regionID := ctx.RegionId
+ key := topic
+ if len(regionID) > 0 {
+ key = fmt.Sprintf("%s%c%s", topic, contentSplitter, regionID)
+ }
+ keyedCtxs[key] = append(keyedCtxs[key], *ctx.marshal2Bean())
+ }
+
+ for k, v := range keyedCtxs {
+ arr := strings.Split(k, string([]byte{contentSplitter}))
+ topic := k
+ regionID := ""
+ if len(arr) > 1 {
+ topic = arr[0]
+ regionID = arr[1]
+ }
+ td.flush(topic, regionID, v)
+ }
+}
+
+type Keyset map[string]struct{}
+
+func (ks Keyset) slice() []string {
+ slice := make([]string, len(ks))
+ for k, _ := range ks {
+ slice = append(slice, k)
+ }
+ return slice
+}
+
+// flush data in batch.
+func (td *traceDispatcher) flush(topic, regionID string, data []TraceTransferBean) {
+ if len(data) == 0 {
+ return
+ }
+
+ keyset := make(Keyset)
+ var builder strings.Builder
+ flushed := true
+ for _, bean := range data {
+ for _, k := range bean.transKey {
+ keyset[k] = struct{}{}
+ }
+ builder.WriteString(bean.transData)
+ flushed = false
+
+ if builder.Len() > maxMsgSize {
+ td.sendTraceDataByMQ(keyset, regionID, builder.String())
+ builder.Reset()
+ keyset = make(Keyset)
+ flushed = true
+ }
+ }
+ if !flushed {
+ td.sendTraceDataByMQ(keyset, regionID, builder.String())
+ }
+}
+
+func (td *traceDispatcher) sendTraceDataByMQ(keySet Keyset, regionID string, data string) {
+ traceTopic := td.traceTopic
+ if td.access == primitive.Cloud {
+ traceTopic = td.traceTopic + regionID
+ }
+ msg := primitive.NewMessage(traceTopic, []byte(data))
+ msg.WithKeys(keySet.slice())
+
+ mq, addr := td.findMq(regionID)
+ if mq == nil {
+ return
+ }
+
+ var req = td.buildSendRequest(mq, msg)
+ ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
+ err := td.cli.InvokeAsync(ctx, addr, req, func(command *remote.RemotingCommand, e error) {
+ resp := new(primitive.SendResult)
+ if e != nil {
+ rlog.Info("send trace data error.", map[string]interface{}{
+ "traceData": data,
+ })
+ } else {
+ td.cli.ProcessSendResponse(mq.BrokerName, command, resp, msg)
+ rlog.Debug("send trace data success:", map[string]interface{}{
+ "SendResult": resp,
+ "traceData": data,
+ })
+ }
+ })
+ if err != nil {
+ rlog.Info("send trace data error when invoke", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+}
+
+func (td *traceDispatcher) findMq(regionID string) (*primitive.MessageQueue, string) {
+ traceTopic := td.traceTopic
+ if td.access == primitive.Cloud {
+ traceTopic = td.traceTopic + regionID
+ }
+ mqs, err := td.namesrvs.FetchPublishMessageQueues(traceTopic)
+ if err != nil {
+ rlog.Error("fetch publish message queues failed", map[string]interface{}{
+ rlog.LogKeyUnderlayError: err,
+ })
+ return nil, ""
+ }
+ i := atomic.AddInt32(&td.rrindex, 1)
+ if i < 0 {
+ i = 0
+ atomic.StoreInt32(&td.rrindex, 0)
+ }
+ i %= int32(len(mqs))
+ mq := mqs[i]
+
+ brokerName := mq.BrokerName
+ addr := td.namesrvs.FindBrokerAddrByName(brokerName)
+
+ return mq, addr
+}
+
+func (td *traceDispatcher) buildSendRequest(mq *primitive.MessageQueue,
+ msg *primitive.Message) *remote.RemotingCommand {
+ req := &SendMessageRequestHeader{
+ ProducerGroup: TraceGroupName,
+ Topic: mq.Topic,
+ QueueId: mq.QueueId,
+ BornTimestamp: time.Now().UnixNano() / int64(time.Millisecond),
+ Flag: msg.Flag,
+ Properties: msg.MarshallProperties(),
+ }
+
+ return remote.NewRemotingCommand(ReqSendMessage, req, msg.Body)
+}
diff --git a/internal/trace_test.go b/internal/trace_test.go
new file mode 100644
index 0000000..9c66f81
--- /dev/null
+++ b/internal/trace_test.go
@@ -0,0 +1,125 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "testing"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMarshal2Bean(t *testing.T) {
+
+ Convey("marshal of TraceContext", t, func() {
+
+ Convey("When marshal producer trace data", func() {
+ traceCtx := TraceContext{
+ TraceType: Pub,
+ TimeStamp: 1563780533299,
+ RegionId: "DefaultRegion",
+ GroupName: "ProducerGroupName",
+ CostTime: 3572,
+ IsSuccess: true,
+ RequestId: "0A5DE93A815518B4AAC26F77F8330001",
+ TraceBeans: []TraceBean{
+ {
+ Topic: "TopicTest",
+ MsgId: "0A5DE93A833B18B4AAC26F842A2F0000",
+ OffsetMsgId: "0A5DE93A00002A9F000000000042E322",
+ Tags: "TagA",
+ Keys: "OrderID1882",
+ StoreHost: "10.93.233.58:10911",
+ ClientHost: "10.93.233.58",
+ StoreTime: 1563780535085,
+ BodyLength: 11,
+ MsgType: primitive.NormalMsg,
+ },
+ },
+ }
+ bean := traceCtx.marshal2Bean()
+ assert.Equal(t, "Pub1563780533299DefaultRegionProducerGroupNameTopicTest0A5DE93A833B18B4AAC26F842A2F0000TagAOrderID188210.93.233.58:1091111357200A5DE93A00002A9F000000000042E322true\x02",
+ bean.transData)
+ assert.Equal(t, []string{"0A5DE93A833B18B4AAC26F842A2F0000", "OrderID1882"}, bean.transKey)
+
+ // consumer before test
+ traceCtx = TraceContext{
+ TraceType: SubBefore,
+ TimeStamp: 1563789119096,
+ GroupName: "CID_JODIE_1",
+ IsSuccess: true,
+ RequestId: "0A5DE93A96A818B4AAC26FFAFA780007",
+ TraceBeans: []TraceBean{
+ {
+ Topic: "TopicTest",
+ MsgId: "0A5DE93A973418B4AAC26FFAFA5A0000",
+ Tags: "TagA",
+ Keys: "OrderID1882",
+ StoreHost: "10.93.233.58",
+ ClientHost: "10.93.233.58",
+ StoreTime: 1563789119092,
+ BodyLength: 190,
+ },
+ },
+ }
+ bean = traceCtx.marshal2Bean()
+
+ Convey("transData should equal to expected", func() {
+ So(bean.transData, ShouldEqual, "SubBefore1563789119096CID_JODIE_10A5DE93A96A818B4AAC26FFAFA7800070A5DE93A973418B4AAC26FFAFA5A00000OrderID1882")
+ })
+
+ Convey("transkey should equal to expected", func() {
+ expectedKey := []string{"0A5DE93A973418B4AAC26FFAFA5A0000", "OrderID1882"}
+ So(bean.transKey[0], ShouldEqual, expectedKey[0])
+ So(bean.transKey[1], ShouldEqual, expectedKey[1])
+ })
+ })
+
+ Convey("When marshal consumer trace data", func() {
+ traceCtx := TraceContext{
+ TraceType: SubAfter,
+ TimeStamp: 1563789119096,
+ GroupName: "CID_JODIE_1",
+ IsSuccess: true,
+ RequestId: "0A5DE93A96A818B4AAC26FFAFA780007",
+ TraceBeans: []TraceBean{
+ {
+ Topic: "TopicTest",
+ MsgId: "0A5DE93A973418B4AAC26FFAFA5A0000",
+ Tags: "TagA",
+ Keys: "OrderID1882",
+ StoreHost: "10.93.233.58",
+ ClientHost: "10.93.233.58",
+ StoreTime: 1563789119092,
+ BodyLength: 190,
+ },
+ },
+ }
+ bean := traceCtx.marshal2Bean()
+ Convey("transData should equal to expected", func() {
+ So(bean.transData, ShouldEqual, "SubAfter0A5DE93A96A818B4AAC26FFAFA7800070A5DE93A973418B4AAC26FFAFA5A00000trueOrderID18820")
+ })
+ Convey("transkey should equal to expected", func() {
+ expectedKey := []string{"0A5DE93A973418B4AAC26FFAFA5A0000", "OrderID1882"}
+ So(bean.transKey[0], ShouldEqual, expectedKey[0])
+ So(bean.transKey[1], ShouldEqual, expectedKey[1])
+ })
+ })
+ })
+}
diff --git a/internal/transaction.go b/internal/transaction.go
new file mode 100644
index 0000000..5cd9835
--- /dev/null
+++ b/internal/transaction.go
@@ -0,0 +1,21 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+type TransactionListener interface {
+}
diff --git a/internal/utils/errors.go b/internal/utils/errors.go
new file mode 100644
index 0000000..0b7ffc2
--- /dev/null
+++ b/internal/utils/errors.go
@@ -0,0 +1,40 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "errors"
+
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+var (
+ // ErrRequestTimeout for request timeout error
+ ErrRequestTimeout = errors.New("request timeout")
+ ErrMQEmpty = errors.New("MessageQueue is nil")
+ ErrOffset = errors.New("offset < 0")
+ ErrNumbers = errors.New("numbers < 0")
+)
+
+func CheckError(action string, err error) {
+ if err != nil {
+ rlog.Error(action, map[string]interface{}{
+ rlog.LogKeyUnderlayError: err.Error(),
+ })
+ }
+}
diff --git a/internal/utils/files.go b/internal/utils/files.go
new file mode 100644
index 0000000..2583d11
--- /dev/null
+++ b/internal/utils/files.go
@@ -0,0 +1,89 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+)
+
+func FileReadAll(path string) ([]byte, error) {
+ stat, err := os.Stat(path)
+ if err != nil {
+ return nil, err
+ }
+ file, err := os.Open(path)
+ if err != nil {
+ return nil, err
+ }
+ data := make([]byte, stat.Size())
+ _, err = file.Read(data)
+ if err != nil {
+ return nil, err
+ }
+ return data, nil
+}
+
+func ensureDir(path string) error {
+ info, err := os.Stat(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ err = os.MkdirAll(path, 0755)
+ }
+ return err
+ }
+ if !info.IsDir() {
+ return errors.New(path + " is a file")
+ }
+ return nil
+}
+
+func WriteToFile(path string, data []byte) error {
+ if err := ensureDir(filepath.Dir(path)); err != nil {
+ return err
+ }
+ tmpFile, err := os.Create(path + ".tmp")
+ if err != nil {
+ return err
+ }
+ _, err = tmpFile.Write(data)
+ if err != nil {
+ return err
+ }
+ CheckError(fmt.Sprintf("close %s", tmpFile.Name()), tmpFile.Close())
+
+ prevContent, err := FileReadAll(path)
+ if err == nil {
+ bakFile, err := os.Create(path + ".bak")
+ if err != nil {
+ _, err = bakFile.Write(prevContent)
+ }
+ if err != nil {
+ return err
+ }
+ CheckError(fmt.Sprintf("close %s", bakFile.Name()), bakFile.Close())
+ }
+
+ _, err = os.Stat(path)
+ if err == nil {
+ CheckError(fmt.Sprintf("remove %s", path), os.Remove(path))
+ }
+ return os.Rename(path+".tmp", path)
+}
diff --git a/internal/utils/helper.go b/internal/utils/helper.go
new file mode 100644
index 0000000..ccadd3f
--- /dev/null
+++ b/internal/utils/helper.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "bytes"
+ "compress/zlib"
+ "io/ioutil"
+ "net"
+)
+
+func GetAddressByBytes(data []byte) string {
+ return net.IPv4(data[0], data[1], data[2], data[3]).String()
+}
+
+func UnCompress(data []byte) []byte {
+ rdata := bytes.NewReader(data)
+ r, err := zlib.NewReader(rdata)
+ if err != nil {
+ return data
+ }
+ retData, err := ioutil.ReadAll(r)
+ if err != nil {
+ return data
+ }
+ return retData
+}
diff --git a/internal/utils/helper_test.go b/internal/utils/helper_test.go
new file mode 100644
index 0000000..837c5ab
--- /dev/null
+++ b/internal/utils/helper_test.go
@@ -0,0 +1,37 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "bytes"
+ "compress/zlib"
+ "testing"
+)
+
+func TestUnCompress(t *testing.T) {
+ var b bytes.Buffer
+ var oriStr string = "hello, go"
+ zr := zlib.NewWriter(&b)
+ zr.Write([]byte(oriStr))
+ zr.Close()
+
+ retBytes := UnCompress(b.Bytes())
+ if string(retBytes) != oriStr {
+ t.Errorf("UnCompress was incorrect, got %s, want: %s .", retBytes, []byte(oriStr))
+ }
+}
diff --git a/internal/utils/math.go b/internal/utils/math.go
new file mode 100644
index 0000000..816631e
--- /dev/null
+++ b/internal/utils/math.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+func AbsInt(i int) int {
+ if i >= 0 {
+ return i
+ }
+ return -i
+}
+
+func MinInt(a, b int) int {
+ if a < b {
+ return a
+ }
+ return b
+}
diff --git a/internal/utils/net.go b/internal/utils/net.go
new file mode 100644
index 0000000..c9444a9
--- /dev/null
+++ b/internal/utils/net.go
@@ -0,0 +1,61 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "net"
+ "strconv"
+ "time"
+)
+
+var (
+ LocalIP string
+)
+
+func init() {
+ ip, err := ClientIP4()
+ if err != nil {
+ LocalIP = ""
+ } else {
+ LocalIP = fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3])
+ }
+}
+
+func ClientIP4() ([]byte, error) {
+ addrs, err := net.InterfaceAddrs()
+ if err != nil {
+ return nil, errors.New("unexpected IP address")
+ }
+ for _, addr := range addrs {
+ if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+ if ip4 := ipnet.IP.To4(); ip4 != nil {
+ return ip4, nil
+ }
+ }
+ }
+ return nil, errors.New("unknown IP address")
+}
+
+func FakeIP() []byte {
+ buf := bytes.NewBufferString("")
+ buf.WriteString(strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
+ return buf.Bytes()[4:8]
+}
diff --git a/internal/utils/net_test.go b/internal/utils/net_test.go
new file mode 100644
index 0000000..3e65a75
--- /dev/null
+++ b/internal/utils/net_test.go
@@ -0,0 +1,24 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import "testing"
+
+func TestLocalIP2(t *testing.T) {
+ t.Log(LocalIP)
+}
diff --git a/internal/utils/set.go b/internal/utils/set.go
new file mode 100644
index 0000000..e90fb36
--- /dev/null
+++ b/internal/utils/set.go
@@ -0,0 +1,103 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+ "bytes"
+ "encoding/json"
+ "sort"
+)
+
+type UniqueItem interface {
+ UniqueID() string
+}
+
+type StringUnique string
+
+func (str StringUnique) UniqueID() string {
+ return string(str)
+}
+
+type Set struct {
+ items map[string]UniqueItem
+}
+
+func NewSet() Set {
+ return Set{
+ items: make(map[string]UniqueItem, 0),
+ }
+}
+
+func (s *Set) Add(v UniqueItem) {
+ s.items[v.UniqueID()] = v
+}
+
+func (s *Set) AddKV(k, v string) {
+ s.items[k] = StringUnique(v)
+}
+
+func (s *Set) Contains(k string) (UniqueItem, bool) {
+ v, ok := s.items[k]
+ return v, ok
+}
+
+func (s *Set) Len() int {
+ return len(s.items)
+}
+
+var _ json.Marshaler = &Set{}
+
+func (s *Set) MarshalJSON() ([]byte, error) {
+ if len(s.items) == 0 {
+ return []byte("[]"), nil
+ }
+
+ buffer := new(bytes.Buffer)
+ buffer.WriteByte('[')
+ keys := make([]string, 0)
+ for _, k := range s.items {
+ var key string
+ switch kval := k.(type) {
+ case StringUnique:
+ key = "\"" + string(kval) + "\""
+ default:
+ v, err := json.Marshal(k)
+ if err != nil {
+ return nil, err
+ }
+ key = string(v)
+ }
+ keys = append(keys, key)
+ }
+ sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
+
+ for i, key := range keys {
+ if i > 0 {
+ buffer.WriteByte(',')
+ }
+ buffer.WriteString(key)
+ }
+
+ buffer.WriteByte(']')
+
+ return buffer.Bytes(), nil
+}
+
+func (s Set) UnmarshalJSON(data []byte) (err error) {
+ return nil
+}
diff --git a/internal/utils/string.go b/internal/utils/string.go
new file mode 100644
index 0000000..f347397
--- /dev/null
+++ b/internal/utils/string.go
@@ -0,0 +1,30 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+// HashString hashes a string to a unique hashcode.
+func HashString(s string) int {
+ val := []byte(s)
+ var h int32
+
+ for idx := range val {
+ h = 31*h + int32(val[idx])
+ }
+
+ return int(h)
+}
diff --git a/internal/validators.go b/internal/validators.go
new file mode 100644
index 0000000..ac51db2
--- /dev/null
+++ b/internal/validators.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internal
+
+import (
+ "regexp"
+
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+const (
+ _ValidPattern = "^[%|a-zA-Z0-9_-]+$"
+ _CharacterMaxLength = 255
+)
+
+var (
+ _Pattern, _ = regexp.Compile(_ValidPattern)
+)
+
+func ValidateGroup(group string) {
+ if group == "" {
+ rlog.Fatal("consumerGroup is empty", nil)
+ }
+
+ if len(group) > _CharacterMaxLength {
+ rlog.Fatal("the specified group is longer than group max length 255.", nil)
+ }
+}
diff --git a/primitive/auth.go b/primitive/auth.go
new file mode 100644
index 0000000..772bc4d
--- /dev/null
+++ b/primitive/auth.go
@@ -0,0 +1,28 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+type Credentials struct {
+ AccessKey string
+ SecretKey string
+ SecurityToken string
+}
+
+func (c Credentials) IsEmpty() bool {
+ return c.AccessKey == "" || c.SecretKey == ""
+}
diff --git a/primitive/base.go b/primitive/base.go
new file mode 100644
index 0000000..35b6268
--- /dev/null
+++ b/primitive/base.go
@@ -0,0 +1,125 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "regexp"
+ "strings"
+)
+
+var (
+ ipRegex, _ = regexp.Compile(`^((25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))`)
+)
+
+type NamesrvAddr []string
+
+func NewNamesrvAddr(s ...string) (NamesrvAddr, error) {
+ if len(s) == 0 {
+ return nil, ErrNoNameserver
+ }
+
+ ss := s
+ if len(ss) == 1 {
+ // compatible with multi server env string: "a;b;c"
+ ss = strings.Split(s[0], ";")
+ }
+
+ for _, srv := range ss {
+ if err := verifyIP(srv); err != nil {
+ return nil, err
+ }
+ }
+
+ addrs := make(NamesrvAddr, 0)
+ addrs = append(addrs, ss...)
+ return addrs, nil
+}
+
+func (addr NamesrvAddr) Check() error {
+ for _, srv := range addr {
+ if err := verifyIP(srv); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+var (
+ httpPrefixRegex, _ = regexp.Compile("^(http|https)://")
+)
+
+func verifyIP(ip string) error {
+ if httpPrefixRegex.MatchString(ip) {
+ return nil
+ }
+ if strings.Contains(ip, ";") {
+ return ErrMultiIP
+ }
+ ips := ipRegex.FindAllString(ip, -1)
+ if len(ips) == 0 {
+ return ErrIllegalIP
+ }
+
+ if len(ips) > 1 {
+ return ErrMultiIP
+ }
+ return nil
+}
+
+var PanicHandler func(interface{})
+
+func WithRecover(fn func()) {
+ defer func() {
+ handler := PanicHandler
+ if handler != nil {
+ if err := recover(); err != nil {
+ handler(err)
+ }
+ }
+ }()
+
+ fn()
+}
+
+func Diff(origin, latest []string) bool {
+ if len(origin) != len(latest) {
+ return true
+ }
+
+ // check added
+ originFilter := make(map[string]struct{}, len(origin))
+ for _, srv := range origin {
+ originFilter[srv] = struct{}{}
+ }
+
+ latestFilter := make(map[string]struct{}, len(latest))
+ for _, srv := range latest {
+ if _, ok := originFilter[srv]; !ok {
+ return true // added
+ }
+ latestFilter[srv] = struct{}{}
+ }
+
+ // check delete
+ for _, srv := range origin {
+ if _, ok := latestFilter[srv]; !ok {
+ return true // deleted
+ }
+ }
+ return false
+}
diff --git a/primitive/base_test.go b/primitive/base_test.go
new file mode 100644
index 0000000..03a978d
--- /dev/null
+++ b/primitive/base_test.go
@@ -0,0 +1,64 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestVerifyIP(t *testing.T) {
+ IPs := "127.0.0.1:9876"
+ err := verifyIP(IPs)
+ assert.Nil(t, err)
+
+ IPs = "12.24.123.243:10911"
+ err = verifyIP(IPs)
+ assert.Nil(t, err)
+
+ IPs = "xa2.0.0.1:9876"
+ err = verifyIP(IPs)
+ assert.Equal(t, "IP addr error", err.Error())
+
+ IPs = "333.0.0.1:9876"
+ err = verifyIP(IPs)
+ assert.Equal(t, "IP addr error", err.Error())
+
+ IPs = "127.0.0.1:9876;12.24.123.243:10911"
+ err = verifyIP(IPs)
+ assert.Equal(t, "multiple IP addr does not support", err.Error())
+}
+
+func TestBase(t *testing.T) {
+ a := []string{}
+ b := []string{}
+ assert.False(t, Diff(a, b))
+
+ a = []string{"a"}
+ b = []string{"a", "b"}
+ assert.True(t, Diff(a, b))
+
+ a = []string{"a", "b", "c"}
+ b = []string{"c", "a", "b"}
+ assert.False(t, Diff(a, b))
+
+ a = []string{"b", "a"}
+ b = []string{"a", "c"}
+ assert.True(t, Diff(a, b))
+}
diff --git a/primitive/ctx.go b/primitive/ctx.go
new file mode 100644
index 0000000..4481dcd
--- /dev/null
+++ b/primitive/ctx.go
@@ -0,0 +1,172 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * Define the ctx key and value type.
+ */
+package primitive
+
+import (
+ "context"
+ "fmt"
+ "math"
+
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+type CtxKey int
+
+type CommunicationMode string
+
+type ConsumeReturnType string
+
+func (c ConsumeReturnType) Ordinal() int {
+ switch c {
+ case SuccessReturn:
+ return 0
+ case TimeoutReturn:
+ return 1
+ case ExceptionReturn:
+ return 2
+ case NullReturn:
+ return 3
+ case FailedReturn:
+ return 4
+ default:
+ rlog.Error(fmt.Sprintf("illegal ConsumeReturnType: %v", c), nil)
+ return 0
+ }
+}
+
+const (
+ method CtxKey = iota
+ msgCtx
+ orderlyCtx
+ concurrentlyCtx
+ producerCtx
+
+ // method name in producer
+ SendSync CommunicationMode = "SendSync"
+ SendOneway CommunicationMode = "SendOneway"
+ SendAsync CommunicationMode = "SendAsync"
+ // method name in consumer
+ ConsumerPush = "ConsumerPush"
+ ConsumerPull = "ConsumerPull"
+
+ PropCtxType = "ConsumeContextType"
+ SuccessReturn ConsumeReturnType = "SUCCESS"
+ TimeoutReturn ConsumeReturnType = "TIMEOUT"
+ ExceptionReturn ConsumeReturnType = "EXCEPTION"
+ NullReturn ConsumeReturnType = "RETURNNULL"
+ FailedReturn ConsumeReturnType = "FAILED"
+)
+
+type ConsumeMessageContext struct {
+ ConsumerGroup string
+ Msgs []*MessageExt
+ MQ *MessageQueue
+ Success bool
+ Status string
+ // mqTractContext
+ Properties map[string]string
+}
+
+// WithMethod set call method name
+func WithMethod(ctx context.Context, m CommunicationMode) context.Context {
+ return context.WithValue(ctx, method, m)
+}
+
+// GetMethod get call method name
+func GetMethod(ctx context.Context) CommunicationMode {
+ return ctx.Value(method).(CommunicationMode)
+}
+
+// WithConsumerCtx set ConsumeMessageContext in PushConsumer
+func WithConsumerCtx(ctx context.Context, c *ConsumeMessageContext) context.Context {
+ return context.WithValue(ctx, msgCtx, c)
+}
+
+// GetConsumerCtx get ConsumeMessageContext, only legal in PushConsumer. so should add bool return param indicate
+// whether exist.
+func GetConsumerCtx(ctx context.Context) (*ConsumeMessageContext, bool) {
+ c, exist := ctx.Value(msgCtx).(*ConsumeMessageContext)
+ return c, exist
+}
+
+type ConsumeOrderlyContext struct {
+ MQ MessageQueue
+ AutoCommit bool
+ SuspendCurrentQueueTimeMillis int
+}
+
+func NewConsumeOrderlyContext() *ConsumeOrderlyContext {
+ return &ConsumeOrderlyContext{
+ AutoCommit: true,
+ SuspendCurrentQueueTimeMillis: -1,
+ }
+}
+
+func WithOrderlyCtx(ctx context.Context, c *ConsumeOrderlyContext) context.Context {
+ return context.WithValue(ctx, orderlyCtx, c)
+}
+
+func GetOrderlyCtx(ctx context.Context) (*ConsumeOrderlyContext, bool) {
+ c, exist := ctx.Value(orderlyCtx).(*ConsumeOrderlyContext)
+ return c, exist
+}
+
+type ConsumeConcurrentlyContext struct {
+ MQ MessageQueue
+ DelayLevelWhenNextConsume int
+ AckIndex int32
+}
+
+func NewConsumeConcurrentlyContext() *ConsumeConcurrentlyContext {
+ return &ConsumeConcurrentlyContext{
+ AckIndex: math.MaxInt32,
+ }
+}
+
+func WithConcurrentlyCtx(ctx context.Context, c *ConsumeConcurrentlyContext) context.Context {
+ return context.WithValue(ctx, concurrentlyCtx, c)
+}
+
+func GetConcurrentlyCtx(ctx context.Context) (*ConsumeConcurrentlyContext, bool) {
+ c, exist := ctx.Value(concurrentlyCtx).(*ConsumeConcurrentlyContext)
+ return c, exist
+}
+
+type ProducerCtx struct {
+ ProducerGroup string
+ Message Message
+ MQ MessageQueue
+ BrokerAddr string
+ BornHost string
+ CommunicationMode CommunicationMode
+ SendResult *SendResult
+ Props map[string]string
+ MsgType MessageType
+ Namespace string
+}
+
+func WithProducerCtx(ctx context.Context, c *ProducerCtx) context.Context {
+ return context.WithValue(ctx, producerCtx, c)
+}
+
+func GetProducerCtx(ctx context.Context) *ProducerCtx {
+ return ctx.Value(producerCtx).(*ProducerCtx)
+}
diff --git a/primitive/errors.go b/primitive/errors.go
new file mode 100644
index 0000000..9b1a88e
--- /dev/null
+++ b/primitive/errors.go
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "errors"
+ "strconv"
+)
+
+var (
+ ErrNoNameserver = errors.New("nameServerAddrs can't be empty.")
+ ErrMultiIP = errors.New("multiple IP addr does not support")
+ ErrIllegalIP = errors.New("IP addr error")
+)
+
+type MQBrokerErr struct {
+ ResponseCode int16
+ ErrorMessage string
+}
+
+func (e MQBrokerErr) Error() string {
+ return "CODE: " + strconv.Itoa(int(e.ResponseCode)) + " DESC: " + e.ErrorMessage
+}
+
+func NewRemotingErr(s string) error {
+ return &RemotingErr{s: s}
+}
+
+type RemotingErr struct {
+ s string
+}
+
+func (e *RemotingErr) Error() string {
+ return e.s
+}
+
+func NewMQClientErr(code int16, msg string) error {
+ return &MQClientErr{code: code, msg: msg}
+}
+
+type MQClientErr struct {
+ code int16
+ msg string
+}
+
+func (e MQClientErr) Error() string {
+ return "CODE: " + strconv.Itoa(int(e.code)) + " DESC: " + e.msg
+}
+
+func IsRemotingErr(err error) bool {
+ _, ok := err.(*RemotingErr)
+ return ok
+}
diff --git a/primitive/interceptor.go b/primitive/interceptor.go
new file mode 100644
index 0000000..878aab5
--- /dev/null
+++ b/primitive/interceptor.go
@@ -0,0 +1,51 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "context"
+)
+
+// Invoker finish a message invoke on producer/consumer.
+type Invoker func(ctx context.Context, req, reply interface{}) error
+
+// Interceptor intercepts the invoke of a producer/consumer on messages.
+// In PushConsumer call, the req is []*MessageExt type and the reply is ConsumeResultHolder,
+// use type assert to get real type.
+type Interceptor func(ctx context.Context, req, reply interface{}, next Invoker) error
+
+func ChainInterceptors(interceptors ...Interceptor) Interceptor {
+ if len(interceptors) == 0 {
+ return nil
+ }
+ if len(interceptors) == 1 {
+ return interceptors[0]
+ }
+ return func(ctx context.Context, req, reply interface{}, invoker Invoker) error {
+ return interceptors[0](ctx, req, reply, getChainedInterceptor(interceptors, 0, invoker))
+ }
+}
+
+func getChainedInterceptor(interceptors []Interceptor, cur int, finalInvoker Invoker) Invoker {
+ if cur == len(interceptors)-1 {
+ return finalInvoker
+ }
+ return func(ctx context.Context, req, reply interface{}) error {
+ return interceptors[cur+1](ctx, req, reply, getChainedInterceptor(interceptors, cur+1, finalInvoker))
+ }
+}
diff --git a/primitive/message.go b/primitive/message.go
new file mode 100644
index 0000000..6a84477
--- /dev/null
+++ b/primitive/message.go
@@ -0,0 +1,546 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "bytes"
+ "encoding/binary"
+ "encoding/hex"
+ "fmt"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+)
+
+const (
+ PropertyKeySeparator = " "
+ PropertyKeys = "KEYS"
+ PropertyTags = "TAGS"
+ PropertyWaitStoreMsgOk = "WAIT"
+ PropertyDelayTimeLevel = "DELAY"
+ PropertyRetryTopic = "RETRY_TOPIC"
+ PropertyRealTopic = "REAL_TOPIC"
+ PropertyRealQueueId = "REAL_QID"
+ PropertyTransactionPrepared = "TRAN_MSG"
+ PropertyProducerGroup = "PGROUP"
+ PropertyMinOffset = "MIN_OFFSET"
+ PropertyMaxOffset = "MAX_OFFSET"
+ PropertyBuyerId = "BUYER_ID"
+ PropertyOriginMessageId = "ORIGIN_MESSAGE_ID"
+ PropertyTransferFlag = "TRANSFER_FLAG"
+ PropertyCorrectionFlag = "CORRECTION_FLAG"
+ PropertyMQ2Flag = "MQ2_FLAG"
+ PropertyReconsumeTime = "RECONSUME_TIME"
+ PropertyMsgRegion = "MSG_REGION"
+ PropertyTraceSwitch = "TRACE_ON"
+ PropertyUniqueClientMessageIdKeyIndex = "UNIQ_KEY"
+ PropertyMaxReconsumeTimes = "MAX_RECONSUME_TIMES"
+ PropertyConsumeStartTime = "CONSUME_START_TIME"
+ PropertyTranscationPreparedQueueOffset = "TRAN_PREPARED_QUEUE_OFFSET"
+ PropertyTranscationCheckTimes = "TRANSACTION_CHECK_TIMES"
+ PropertyCheckImmunityTimeInSeconds = "CHECK_IMMUNITY_TIME_IN_SECONDS"
+ PropertyShardingKey = "SHARDING_KEY"
+)
+
+type Message struct {
+ Topic string
+ Body []byte
+ Flag int32
+ TransactionId string
+ Batch bool
+ // Queue is the queue that messages will be sent to. the value must be set if want to custom the queue of message,
+ // just ignore if not.
+ Queue *MessageQueue
+
+ properties map[string]string
+ mutex sync.RWMutex
+}
+
+func (m *Message) WithProperties(p map[string]string) {
+ m.mutex.Lock()
+ m.properties = p
+ m.mutex.Unlock()
+}
+
+func (m *Message) WithProperty(key, value string) {
+ if key == "" || value == "" {
+ return
+ }
+ m.mutex.Lock()
+ if m.properties == nil {
+ m.properties = make(map[string]string)
+ }
+ m.properties[key] = value
+ m.mutex.Unlock()
+}
+
+func (m *Message) GetProperty(key string) string {
+ m.mutex.RLock()
+ v := m.properties[key]
+ m.mutex.RUnlock()
+ return v
+}
+
+func (m *Message) RemoveProperty(key string) string {
+ m.mutex.Lock()
+ defer m.mutex.Unlock()
+ value, exist := m.properties[key]
+ if !exist {
+ return ""
+ }
+ delete(m.properties, key)
+ return value
+}
+func (m *Message) MarshallProperties() string {
+ m.mutex.RLock()
+ defer m.mutex.RUnlock()
+ buffer := bytes.NewBufferString("")
+ for k, v := range m.properties {
+ buffer.WriteString(k)
+ buffer.WriteRune(nameValueSeparator)
+ buffer.WriteString(v)
+ buffer.WriteRune(propertySeparator)
+ }
+ return buffer.String()
+}
+
+// unmarshalProperties parse data into property kv pairs.
+func (m *Message) UnmarshalProperties(data []byte) {
+ m.mutex.Lock()
+ defer m.mutex.Unlock()
+ if m.properties == nil {
+ m.properties = make(map[string]string)
+ }
+ items := bytes.Split(data, []byte{propertySeparator})
+ for _, item := range items {
+ kv := bytes.Split(item, []byte{nameValueSeparator})
+ if len(kv) == 2 {
+ m.properties[string(kv[0])] = string(kv[1])
+ }
+ }
+}
+
+func (m *Message) GetProperties() map[string]string {
+ m.mutex.Lock()
+ defer m.mutex.Unlock()
+ result := make(map[string]string, len(m.properties))
+ for k, v := range m.properties {
+ result[k] = v
+ }
+ return result
+}
+
+func NewMessage(topic string, body []byte) *Message {
+ msg := &Message{
+ Topic: topic,
+ Body: body,
+ properties: make(map[string]string),
+ }
+ //msg.properties[PropertyWaitStoreMsgOk] = strconv.FormatBool(true)
+ return msg
+}
+
+// WithDelayTimeLevel set message delay time to consume.
+// reference delay level definition: 1s 5s 10s 30s 1m 2m 3m 4m 5m 6m 7m 8m 9m 10m 20m 30m 1h 2h
+// delay level starts from 1. for example, if we set param level=1, then the delay time is 1s.
+func (m *Message) WithDelayTimeLevel(level int) *Message {
+ m.WithProperty(PropertyDelayTimeLevel, strconv.Itoa(level))
+ return m
+}
+
+func (m *Message) WithTag(tags string) *Message {
+ m.WithProperty(PropertyTags, tags)
+ return m
+}
+
+func (m *Message) WithKeys(keys []string) *Message {
+ var sb strings.Builder
+ for _, k := range keys {
+ sb.WriteString(k)
+ sb.WriteString(PropertyKeySeparator)
+ }
+
+ m.WithProperty(PropertyKeys, sb.String())
+ return m
+}
+
+func (m *Message) WithShardingKey(key string) *Message {
+ m.WithProperty(PropertyShardingKey, key)
+ return m
+}
+
+func (m *Message) GetTags() string {
+ return m.GetProperty(PropertyTags)
+}
+
+func (m *Message) GetKeys() string {
+ return m.GetProperty(PropertyKeys)
+}
+
+func (m *Message) GetShardingKey() string {
+ return m.GetProperty(PropertyShardingKey)
+}
+
+func (m *Message) String() string {
+ return fmt.Sprintf("[topic=%s, body=%s, Flag=%d, properties=%v, TransactionId=%s]",
+ m.Topic, string(m.Body), m.Flag, m.properties, m.TransactionId)
+}
+
+func (m *Message) Marshal() []byte {
+ // storeSize all size of message info.
+ // TOTALSIZE MAGICCOD BODYCRC FLAG BODYSIZE BODY PROPERTYSIZE PROPERTY
+ v := m.MarshallProperties()
+ properties := []byte(v)
+ storeSize := 4 + 4 + 4 + 4 + 4 + len(m.Body) + 2 + len(properties)
+
+ buffer := make([]byte, storeSize)
+ pos := 0
+ binary.BigEndian.PutUint32(buffer[pos:], uint32(storeSize)) // 1. TOTALSIZE
+ pos += 4
+ binary.BigEndian.PutUint32(buffer[pos:], 0) // 2. MAGICCODE
+ pos += 4
+ binary.BigEndian.PutUint32(buffer[pos:], 0) // 3. BODYCRC
+ pos += 4
+ binary.BigEndian.PutUint32(buffer[pos:], uint32(m.Flag)) // 4. FLAG
+ pos += 4
+ binary.BigEndian.PutUint32(buffer[pos:], uint32(len(m.Body))) // 5. BODYSIZE
+ pos += 4
+ copy(buffer[pos:], m.Body)
+ pos += len(m.Body)
+
+ binary.BigEndian.PutUint16(buffer[pos:], uint16(len(properties))) // 7. PROPERTYSIZE
+ pos += 2
+ copy(buffer[pos:], properties)
+
+ return buffer
+}
+
+type MessageExt struct {
+ Message
+ MsgId string
+ OffsetMsgId string
+ StoreSize int32
+ QueueOffset int64
+ SysFlag int32
+ BornTimestamp int64
+ BornHost string
+ StoreTimestamp int64
+ StoreHost string
+ CommitLogOffset int64
+ BodyCRC int32
+ ReconsumeTimes int32
+ PreparedTransactionOffset int64
+}
+
+func (msgExt *MessageExt) GetTags() string {
+ return msgExt.GetProperty(PropertyTags)
+}
+
+func (msgExt *MessageExt) GetRegionID() string {
+ return msgExt.GetProperty(PropertyMsgRegion)
+}
+
+func (msgExt *MessageExt) IsTraceOn() string {
+ return msgExt.GetProperty(PropertyTraceSwitch)
+}
+
+func (msgExt *MessageExt) String() string {
+ return fmt.Sprintf("[Message=%s, MsgId=%s, OffsetMsgId=%s,QueueId=%d, StoreSize=%d, QueueOffset=%d, SysFlag=%d, "+
+ "BornTimestamp=%d, BornHost=%s, StoreTimestamp=%d, StoreHost=%s, CommitLogOffset=%d, BodyCRC=%d, "+
+ "ReconsumeTimes=%d, PreparedTransactionOffset=%d]", msgExt.Message.String(), msgExt.MsgId, msgExt.OffsetMsgId, msgExt.Queue.QueueId,
+ msgExt.StoreSize, msgExt.QueueOffset, msgExt.SysFlag, msgExt.BornTimestamp, msgExt.BornHost,
+ msgExt.StoreTimestamp, msgExt.StoreHost, msgExt.CommitLogOffset, msgExt.BodyCRC, msgExt.ReconsumeTimes,
+ msgExt.PreparedTransactionOffset)
+}
+
+func DecodeMessage(data []byte) []*MessageExt {
+ msgs := make([]*MessageExt, 0)
+ buf := bytes.NewBuffer(data)
+ count := 0
+ for count < len(data) {
+ msg := &MessageExt{}
+ msg.Queue = &MessageQueue{}
+
+ // 1. total size
+ binary.Read(buf, binary.BigEndian, &msg.StoreSize)
+ count += 4
+
+ // 2. magic code
+ buf.Next(4)
+ count += 4
+
+ // 3. body CRC32
+ binary.Read(buf, binary.BigEndian, &msg.BodyCRC)
+ count += 4
+
+ // 4. queueID
+ var qId int32
+ binary.Read(buf, binary.BigEndian, &qId)
+ msg.Queue.QueueId = int(qId)
+ count += 4
+
+ // 5. Flag
+ binary.Read(buf, binary.BigEndian, &msg.Flag)
+ count += 4
+
+ // 6. QueueOffset
+ binary.Read(buf, binary.BigEndian, &msg.QueueOffset)
+ count += 8
+
+ // 7. physical offset
+ binary.Read(buf, binary.BigEndian, &msg.CommitLogOffset)
+ count += 8
+
+ // 8. SysFlag
+ binary.Read(buf, binary.BigEndian, &msg.SysFlag)
+ count += 4
+
+ // 9. BornTimestamp
+ binary.Read(buf, binary.BigEndian, &msg.BornTimestamp)
+ count += 8
+
+ // 10. born host
+ hostBytes := buf.Next(4)
+ var port int32
+ binary.Read(buf, binary.BigEndian, &port)
+ msg.BornHost = fmt.Sprintf("%s:%d", utils.GetAddressByBytes(hostBytes), port)
+ count += 8
+
+ // 11. store timestamp
+ binary.Read(buf, binary.BigEndian, &msg.StoreTimestamp)
+ count += 8
+
+ // 12. store host
+ hostBytes = buf.Next(4)
+ binary.Read(buf, binary.BigEndian, &port)
+ msg.StoreHost = fmt.Sprintf("%s:%d", utils.GetAddressByBytes(hostBytes), port)
+ count += 8
+
+ // 13. reconsume times
+ binary.Read(buf, binary.BigEndian, &msg.ReconsumeTimes)
+ count += 4
+
+ // 14. prepared transaction offset
+ binary.Read(buf, binary.BigEndian, &msg.PreparedTransactionOffset)
+ count += 8
+
+ // 15. body
+ var length int32
+ binary.Read(buf, binary.BigEndian, &length)
+ msg.Body = buf.Next(int(length))
+ if (msg.SysFlag & FlagCompressed) == FlagCompressed {
+ msg.Body = utils.UnCompress(msg.Body)
+ }
+ count += 4 + int(length)
+
+ // 16. topic
+ _byte, _ := buf.ReadByte()
+ msg.Topic = string(buf.Next(int(_byte)))
+ count += 1 + int(_byte)
+
+ // 17. properties
+ var propertiesLength int16
+ binary.Read(buf, binary.BigEndian, &propertiesLength)
+ if propertiesLength > 0 {
+ msg.UnmarshalProperties(buf.Next(int(propertiesLength)))
+ }
+ count += 2 + int(propertiesLength)
+
+ msg.OffsetMsgId = CreateMessageId(hostBytes, port, msg.CommitLogOffset)
+ //count += 16
+ if msg.properties == nil {
+ msg.properties = make(map[string]string, 0)
+ }
+ msgID := msg.GetProperty(PropertyUniqueClientMessageIdKeyIndex)
+ if len(msgID) == 0 {
+ msg.MsgId = msg.OffsetMsgId
+ } else {
+ msg.MsgId = msgID
+ }
+ msgs = append(msgs, msg)
+ }
+
+ return msgs
+}
+
+// MessageQueue message queue
+type MessageQueue struct {
+ Topic string `json:"topic"`
+ BrokerName string `json:"brokerName"`
+ QueueId int `json:"queueId"`
+}
+
+func (mq *MessageQueue) String() string {
+ return fmt.Sprintf("MessageQueue [topic=%s, brokerName=%s, queueId=%d]", mq.Topic, mq.BrokerName, mq.QueueId)
+}
+
+func (mq *MessageQueue) HashCode() int {
+ result := 1
+ result = 31*result + utils.HashString(mq.BrokerName)
+ result = 31*result + mq.QueueId
+ result = 31*result + utils.HashString(mq.Topic)
+
+ return result
+}
+
+type AccessChannel int
+
+const (
+ // connect to private IDC cluster.
+ Local AccessChannel = iota
+ // connect to Cloud service.
+ Cloud
+)
+
+type MessageType int
+
+const (
+ NormalMsg MessageType = iota
+ TransMsgHalf
+ TransMsgCommit
+ DelayMsg
+)
+
+type LocalTransactionState int
+
+const (
+ CommitMessageState LocalTransactionState = iota + 1
+ RollbackMessageState
+ UnknowState
+)
+
+type TransactionListener interface {
+ // When send transactional prepare(half) message succeed, this method will be invoked to execute local transaction.
+ ExecuteLocalTransaction(*Message) LocalTransactionState
+
+ // When no response to prepare(half) message. broker will send check message to check the transaction status, and this
+ // method will be invoked to get local transaction status.
+ CheckLocalTransaction(*MessageExt) LocalTransactionState
+}
+
+type MessageID struct {
+ Addr string
+ Port int
+ Offset int64
+}
+
+func CreateMessageId(addr []byte, port int32, offset int64) string {
+ buffer := GetBuffer()
+ defer BackBuffer(buffer)
+ buffer.Write(addr)
+ _ = binary.Write(buffer, binary.BigEndian, port)
+ _ = binary.Write(buffer, binary.BigEndian, offset)
+ return strings.ToUpper(hex.EncodeToString(buffer.Bytes()))
+}
+
+func UnmarshalMsgID(id []byte) (*MessageID, error) {
+ if len(id) < 32 {
+ return nil, fmt.Errorf("%s len < 32", string(id))
+ }
+ var (
+ ipBytes = make([]byte, 4)
+ portBytes = make([]byte, 4)
+ offsetBytes = make([]byte, 8)
+ )
+ hex.Decode(ipBytes, id[0:8])
+ hex.Decode(portBytes, id[8:16])
+ hex.Decode(offsetBytes, id[16:32])
+
+ return &MessageID{
+ Addr: utils.GetAddressByBytes(ipBytes),
+ Port: int(binary.BigEndian.Uint32(portBytes)),
+ Offset: int64(binary.BigEndian.Uint64(offsetBytes)),
+ }, nil
+}
+
+var (
+ CompressedFlag = 0x1
+
+ MultiTagsFlag = 0x1 << 1
+
+ TransactionNotType = 0
+
+ TransactionPreparedType = 0x1 << 2
+
+ TransactionCommitType = 0x2 << 2
+
+ TransactionRollbackType = 0x3 << 2
+)
+
+func GetTransactionValue(flag int) int {
+ return flag & TransactionRollbackType
+}
+
+func ResetTransactionValue(flag int, typeFlag int) int {
+ return (flag & (^TransactionRollbackType)) | typeFlag
+}
+
+func ClearCompressedFlag(flag int) int {
+ return flag & (^CompressedFlag)
+}
+
+var (
+ counter int16 = 0
+ startTimestamp int64 = 0
+ nextTimestamp int64 = 0
+ prefix string
+ locker sync.Mutex
+ classLoadId int32 = 0
+)
+
+func init() {
+ buf := new(bytes.Buffer)
+
+ ip, err := utils.ClientIP4()
+ if err != nil {
+ ip = utils.FakeIP()
+ }
+ _, _ = buf.Write(ip)
+ _ = binary.Write(buf, binary.BigEndian, Pid())
+ _ = binary.Write(buf, binary.BigEndian, classLoadId)
+ prefix = strings.ToUpper(hex.EncodeToString(buf.Bytes()))
+}
+
+func CreateUniqID() string {
+ locker.Lock()
+ defer locker.Unlock()
+
+ if time.Now().Unix() > nextTimestamp {
+ updateTimestamp()
+ }
+ counter++
+ buf := new(bytes.Buffer)
+ _ = binary.Write(buf, binary.BigEndian, int32((time.Now().Unix()-startTimestamp)*1000))
+ _ = binary.Write(buf, binary.BigEndian, counter)
+
+ return prefix + hex.EncodeToString(buf.Bytes())
+}
+
+func updateTimestamp() {
+ year, month := time.Now().Year(), time.Now().Month()
+ startTimestamp = time.Date(year, month, 1, 0, 0, 0, 0, time.Local).Unix()
+ nextTimestamp = time.Date(year, month, 1, 0, 0, 0, 0, time.Local).AddDate(0, 1, 0).Unix()
+}
+
+func Pid() int16 {
+ return int16(os.Getpid())
+}
diff --git a/primitive/message_test.go b/primitive/message_test.go
new file mode 100644
index 0000000..e47a298
--- /dev/null
+++ b/primitive/message_test.go
@@ -0,0 +1,38 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import "testing"
+
+func TestMessageID(t *testing.T) {
+ id := []byte("0AAF0895000078BF000000000009BB4A")
+ msgID, err := UnmarshalMsgID(id)
+ if err != nil {
+ t.Fatalf("unmarshal msg id error, ms is: %s", err.Error())
+ }
+ if msgID.Addr != "10.175.8.149" {
+ t.Fatalf("parse messageID %s error", id)
+ }
+ if msgID.Port != 30911 {
+ t.Fatalf("parse messageID %s error", id)
+ }
+ if msgID.Offset != 637770 {
+ t.Fatalf("parse messageID %s error", id)
+ }
+ t.Log(msgID)
+}
diff --git a/primitive/nsresolver.go b/primitive/nsresolver.go
new file mode 100644
index 0000000..d844373
--- /dev/null
+++ b/primitive/nsresolver.go
@@ -0,0 +1,219 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package primitive
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "os/user"
+ "path"
+ "strings"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+// resolver for nameserver, monitor change of nameserver and notify client
+// consul or domain is common
+type NsResolver interface {
+ Resolve() []string
+ Description() string
+}
+
+type StaticResolver struct {
+}
+
+var _ NsResolver = (*EnvResolver)(nil)
+
+func NewEnvResolver() *EnvResolver {
+ return &EnvResolver{}
+}
+
+type EnvResolver struct {
+}
+
+func (e *EnvResolver) Resolve() []string {
+ if v := os.Getenv("NAMESRV_ADDR"); v != "" {
+ return strings.Split(v, ";")
+ }
+ return nil
+}
+
+func (e *EnvResolver) Description() string {
+ return "env resolver of var NAMESRV_ADDR"
+}
+
+type passthroughResolver struct {
+ addr []string
+ failback NsResolver
+}
+
+func NewPassthroughResolver(addr []string) *passthroughResolver {
+ return &passthroughResolver{
+ addr: addr,
+ failback: NewEnvResolver(),
+ }
+}
+
+func (p *passthroughResolver) Resolve() []string {
+ if p.addr != nil {
+ return p.addr
+ }
+ return p.failback.Resolve()
+}
+
+func (p *passthroughResolver) Description() string {
+ return fmt.Sprintf("passthrough resolver of %v", p.addr)
+}
+
+const (
+ DEFAULT_NAMESRV_ADDR = "http://jmenv.tbsite.net:8080/rocketmq/nsaddr"
+)
+
+var _ NsResolver = (*HttpResolver)(nil)
+
+type HttpResolver struct {
+ domain string
+ instance string
+ cli http.Client
+ failback NsResolver
+}
+
+func NewHttpResolver(instance string, domain ...string) *HttpResolver {
+ d := DEFAULT_NAMESRV_ADDR
+ if len(domain) > 0 {
+ d = domain[0]
+ }
+ client := http.Client{Timeout: 10 * time.Second}
+
+ h := &HttpResolver{
+ domain: d,
+ instance: instance,
+ cli: client,
+ failback: NewEnvResolver(),
+ }
+ return h
+}
+
+func (h *HttpResolver) Resolve() []string {
+ addrs := h.get()
+ if len(addrs) > 0 {
+ return addrs
+ }
+
+ addrs = h.loadSnapshot()
+ if len(addrs) > 0 {
+ return addrs
+ }
+ return h.failback.Resolve()
+}
+
+func (h *HttpResolver) Description() string {
+ return fmt.Sprintf("passthrough resolver of domain:%v instance:%v", h.domain, h.instance)
+}
+
+func (h *HttpResolver) get() []string {
+ resp, err := h.cli.Get(h.domain)
+ if err != nil {
+ rlog.Error("name server http fetch failed", map[string]interface{}{
+ "NameServerDomain": h.domain,
+ "err": err,
+ })
+ return nil
+ }
+
+ defer resp.Body.Close()
+ body, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ rlog.Error("name server read http response failed", map[string]interface{}{
+ "NameServerDomain": h.domain,
+ "err": err,
+ })
+ return nil
+ }
+
+ bodyStr := string(body)
+ if bodyStr == "" {
+ return nil
+ }
+
+ h.saveSnapshot(body)
+
+ return strings.Split(string(body), ";")
+}
+
+func (h *HttpResolver) saveSnapshot(body []byte) error {
+ filePath := h.getSnapshotFilePath(h.instance)
+ err := ioutil.WriteFile(filePath, body, 0644)
+ if err != nil {
+ rlog.Error("name server snapshot save failed", map[string]interface{}{
+ "filePath": filePath,
+ "err": err,
+ })
+ return err
+ }
+
+ rlog.Info("name server snapshot save successfully", map[string]interface{}{
+ "filePath": filePath,
+ })
+ return nil
+}
+
+func (h *HttpResolver) loadSnapshot() []string {
+ filePath := h.getSnapshotFilePath(h.instance)
+ _, err := os.Stat(filePath)
+ if os.IsNotExist(err) {
+ rlog.Warning("name server snapshot local file not exists", map[string]interface{}{
+ "filePath": filePath,
+ })
+ return nil
+ }
+
+ bs, err := ioutil.ReadFile(filePath)
+ if err != nil {
+ return nil
+ }
+
+ rlog.Info("load the name server snapshot local file", map[string]interface{}{
+ "filePath": filePath,
+ })
+ return strings.Split(string(bs), ";")
+}
+
+func (h *HttpResolver) getSnapshotFilePath(instanceName string) string {
+ homeDir := ""
+ if usr, err := user.Current(); err == nil {
+ homeDir = usr.HomeDir
+ } else {
+ rlog.Error("name server domain, can't get user home directory", map[string]interface{}{
+ "err": err,
+ })
+ }
+ storePath := path.Join(homeDir, "/logs/rocketmq-go/snapshot")
+ if _, err := os.Stat(storePath); os.IsNotExist(err) {
+ if err = os.MkdirAll(storePath, 0755); err != nil {
+ rlog.Fatal("can't create name server snapshot directory", map[string]interface{}{
+ "path": storePath,
+ "err": err,
+ })
+ }
+ }
+ filePath := path.Join(storePath, fmt.Sprintf("nameserver_addr-%s", instanceName))
+ return filePath
+}
diff --git a/primitive/nsresolver_test.go b/primitive/nsresolver_test.go
new file mode 100644
index 0000000..98d839a
--- /dev/null
+++ b/primitive/nsresolver_test.go
@@ -0,0 +1,133 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package primitive
+
+import (
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "os"
+ "strings"
+ "testing"
+
+ . "github.com/smartystreets/goconvey/convey"
+)
+
+func TestEnvResolver(t *testing.T) {
+ Convey("Test UpdateNameServerAddress Use Env", t, func() {
+ srvs := []string{
+ "192.168.100.1",
+ "192.168.100.2",
+ "192.168.100.3",
+ "192.168.100.4",
+ "192.168.100.5",
+ }
+
+ resolver := NewEnvResolver()
+ os.Setenv("NAMESRV_ADDR", strings.Join(srvs, ";"))
+
+ addrs := resolver.Resolve()
+
+ So(Diff(srvs, addrs), ShouldBeFalse)
+ })
+}
+
+func TestHttpResolverWithGet(t *testing.T) {
+ Convey("Test UpdateNameServerAddress Save Local Snapshot", t, func() {
+ srvs := []string{
+ "192.168.100.1",
+ "192.168.100.2",
+ "192.168.100.3",
+ "192.168.100.4",
+ "192.168.100.5",
+ }
+ http.HandleFunc("/nameserver/addrs2", func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprintf(w, strings.Join(srvs, ";"))
+ })
+ server := &http.Server{Addr: ":0", Handler: nil}
+ listener, _ := net.Listen("tcp", ":0")
+ go server.Serve(listener)
+
+ port := listener.Addr().(*net.TCPAddr).Port
+ nameServerDommain := fmt.Sprintf("http://127.0.0.1:%d/nameserver/addrs2", port)
+ fmt.Println("temporary name server domain: ", nameServerDommain)
+
+ resolver := NewHttpResolver("DEFAULT", nameServerDommain)
+ resolver.Resolve()
+
+ // check snapshot saved
+ filePath := resolver.getSnapshotFilePath("DEFAULT")
+ body := strings.Join(srvs, ";")
+ bs, _ := ioutil.ReadFile(filePath)
+ So(string(bs), ShouldEqual, body)
+ })
+}
+
+func TestHttpResolverWithSnapshotFile(t *testing.T) {
+ Convey("Test UpdateNameServerAddress Use Local Snapshot", t, func() {
+ srvs := []string{
+ "192.168.100.1",
+ "192.168.100.2",
+ "192.168.100.3",
+ "192.168.100.4",
+ "192.168.100.5",
+ }
+
+ resolver := NewHttpResolver("DEFAULT", "http://127.0.0.1:80/error/nsaddrs")
+
+ os.Setenv("NAMESRV_ADDR", "") // clear env
+ // setup local snapshot file
+ filePath := resolver.getSnapshotFilePath("DEFAULT")
+ body := strings.Join(srvs, ";")
+ _ = ioutil.WriteFile(filePath, []byte(body), 0644)
+
+ addrs := resolver.Resolve()
+
+ So(Diff(addrs, srvs), ShouldBeFalse)
+ })
+}
+
+func TesHttpReslverWithSnapshotFileOnce(t *testing.T) {
+ Convey("Test UpdateNameServerAddress Load Local Snapshot Once", t, func() {
+ srvs := []string{
+ "192.168.100.1",
+ "192.168.100.2",
+ "192.168.100.3",
+ "192.168.100.4",
+ "192.168.100.5",
+ }
+
+ resolver := NewHttpResolver("DEFAULT", "http://127.0.0.1:80/error/nsaddrs")
+
+ os.Setenv("NAMESRV_ADDR", "") // clear env
+ // setup local snapshot file
+ filePath := resolver.getSnapshotFilePath("DEFAULT")
+ body := strings.Join(srvs, ";")
+ _ = ioutil.WriteFile(filePath, []byte(body), 0644)
+ // load local snapshot file first time
+ addrs1 := resolver.Resolve()
+
+ // change the local snapshot file to check load once
+ _ = ioutil.WriteFile(filePath, []byte("127.0.0.1;127.0.0.2"), 0644)
+
+ addrs2 := resolver.Resolve()
+
+ So(Diff(addrs1, addrs2), ShouldBeFalse)
+ So(Diff(addrs1, srvs), ShouldBeFalse)
+ })
+}
diff --git a/primitive/pool.go b/primitive/pool.go
new file mode 100644
index 0000000..946c18e
--- /dev/null
+++ b/primitive/pool.go
@@ -0,0 +1,56 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "bytes"
+ "sync"
+)
+
+var headerPool = sync.Pool{}
+var bufferPool = sync.Pool{}
+
+func init() {
+ headerPool.New = func() interface{} {
+ return make([]byte, 4)
+ }
+ bufferPool.New = func() interface{} {
+ return new(bytes.Buffer)
+ }
+}
+
+func GetHeader() []byte {
+ d := headerPool.Get().([]byte)
+ //d = (d)[:0]
+ return d
+}
+
+func BackHeader(d []byte) {
+ headerPool.Put(d)
+}
+
+func GetBuffer() *bytes.Buffer {
+ b := bufferPool.Get().(*bytes.Buffer)
+ b.Reset()
+ return b
+}
+
+func BackBuffer(b *bytes.Buffer) {
+ b.Reset()
+ bufferPool.Put(b)
+}
diff --git a/primitive/result.go b/primitive/result.go
new file mode 100644
index 0000000..0d96454
--- /dev/null
+++ b/primitive/result.go
@@ -0,0 +1,122 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "fmt"
+)
+
+// SendStatus of message
+type SendStatus int
+
+const (
+ SendOK SendStatus = iota
+ SendFlushDiskTimeout
+ SendFlushSlaveTimeout
+ SendSlaveNotAvailable
+ SendUnknownError
+
+ FlagCompressed = 0x1
+ MsgIdLength = 8 + 8
+
+ propertySeparator = '\002'
+ nameValueSeparator = '\001'
+)
+
+// SendResult RocketMQ send result
+type SendResult struct {
+ Status SendStatus
+ MsgID string
+ MessageQueue *MessageQueue
+ QueueOffset int64
+ TransactionID string
+ OffsetMsgID string
+ RegionID string
+ TraceOn bool
+}
+
+// SendResult send message result to string(detail result)
+func (result *SendResult) String() string {
+ return fmt.Sprintf("SendResult [sendStatus=%d, msgIds=%s, offsetMsgId=%s, queueOffset=%d, messageQueue=%s]",
+ result.Status, result.MsgID, result.OffsetMsgID, result.QueueOffset, result.MessageQueue.String())
+}
+
+// SendResult RocketMQ send result
+type TransactionSendResult struct {
+ *SendResult
+ State LocalTransactionState
+}
+
+// PullStatus pull Status
+type PullStatus int
+
+// predefined pull Status
+const (
+ PullFound PullStatus = iota
+ PullNoNewMsg
+ PullNoMsgMatched
+ PullOffsetIllegal
+ PullBrokerTimeout
+)
+
+// PullResult the pull result
+type PullResult struct {
+ NextBeginOffset int64
+ MinOffset int64
+ MaxOffset int64
+ Status PullStatus
+ SuggestWhichBrokerId int64
+
+ // messageExts message info
+ messageExts []*MessageExt
+ //
+ body []byte
+}
+
+func (result *PullResult) GetMessageExts() []*MessageExt {
+ return result.messageExts
+}
+
+func (result *PullResult) SetMessageExts(msgExts []*MessageExt) {
+ result.messageExts = msgExts
+}
+
+func (result *PullResult) GetMessages() []*Message {
+ if len(result.messageExts) == 0 {
+ return make([]*Message, 0)
+ }
+ return toMessages(result.messageExts)
+}
+
+func (result *PullResult) SetBody(data []byte) {
+ result.body = data
+}
+
+func (result *PullResult) GetBody() []byte {
+ return result.body
+}
+
+func (result *PullResult) String() string {
+ return ""
+}
+
+func toMessages(messageExts []*MessageExt) []*Message {
+ msgs := make([]*Message, 0)
+
+ return msgs
+}
diff --git a/primitive/result_test.go b/primitive/result_test.go
new file mode 100644
index 0000000..fcc03cf
--- /dev/null
+++ b/primitive/result_test.go
@@ -0,0 +1,74 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+import (
+ "testing"
+
+ . "github.com/smartystreets/goconvey/convey"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestProperties(t *testing.T) {
+ msg1 := NewMessage("test", nil)
+ msg1.properties = map[string]string{
+ "k1": "v1",
+ "k2": "v2",
+ }
+ str := msg1.MarshallProperties()
+ msg2 := NewMessage("test", nil)
+ msg2.UnmarshalProperties([]byte(str))
+ assert.Equal(t, msg1.properties, msg2.properties)
+}
+
+func TestCreateMessageId(t *testing.T) {
+ Convey("MessageId gen", t, func() {
+ b := []byte{10, 93, 233, 58}
+ port := int32(10911)
+ offset := int64(4391252)
+ id := CreateMessageId(b, port, offset)
+ Convey("generated messageId should be equal to expected", func() {
+ assert.Equal(t, "0A5DE93A00002A9F0000000000430154", id)
+ })
+
+ b2 := []byte("127.0.0.1")
+ port2 := int32(11)
+ offset2 := int64(12)
+ id2 := CreateMessageId(b2, port2, offset2)
+ Convey("new generated messageId should be equal to expected", func() {
+ assert.Equal(t, "3132372E302E302E310000000B000000000000000C", id2)
+ })
+
+ Convey("ex-generated messageId should not change", func() {
+ assert.Equal(t, "0A5DE93A00002A9F0000000000430154", id)
+ })
+ })
+
+}
+
+func TestGetProperties(t *testing.T) {
+ msg1 := NewMessage("test", nil)
+ msg1.properties = map[string]string{
+ "k1": "v1",
+ "k2": "v2",
+ }
+ assert.Equal(t, msg1.GetProperties(), map[string]string{
+ "k1": "v1",
+ "k2": "v2",
+ })
+}
diff --git a/primitive/trace.go b/primitive/trace.go
new file mode 100644
index 0000000..c0df5b3
--- /dev/null
+++ b/primitive/trace.go
@@ -0,0 +1,27 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package primitive
+
+// config for message trace.
+type TraceConfig struct {
+ TraceTopic string
+ GroupName string
+ Access AccessChannel
+ NamesrvAddrs []string
+ Credentials // acl config for trace. omit if acl is closed on broker.
+}
diff --git a/producer/interceptor.go b/producer/interceptor.go
new file mode 100644
index 0000000..b3d2598
--- /dev/null
+++ b/producer/interceptor.go
@@ -0,0 +1,97 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * builtin interceptor
+ */
+package producer
+
+import (
+ "context"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+// WithTrace support rocketmq trace: https://github.com/apache/rocketmq/wiki/RIP-6-Message-Trace.
+func WithTrace(traceCfg *primitive.TraceConfig) Option {
+ return func(options *producerOptions) {
+
+ ori := options.Interceptors
+ options.Interceptors = make([]primitive.Interceptor, 0)
+ options.Interceptors = append(options.Interceptors, newTraceInterceptor(traceCfg))
+ options.Interceptors = append(options.Interceptors, ori...)
+ }
+}
+
+func newTraceInterceptor(traceCfg *primitive.TraceConfig) primitive.Interceptor {
+ dispatcher := internal.NewTraceDispatcher(traceCfg)
+ dispatcher.Start()
+
+ return func(ctx context.Context, req, reply interface{}, next primitive.Invoker) error {
+ beginT := time.Now()
+ err := next(ctx, req, reply)
+
+ producerCtx := primitive.GetProducerCtx(ctx)
+ if producerCtx.Message.Topic == dispatcher.GetTraceTopicName() {
+ return next(ctx, req, reply)
+ }
+
+ // SendOneway && SendAsync has no reply.
+ if reply == nil {
+ return err
+ }
+
+ result := reply.(*primitive.SendResult)
+ if result.RegionID == "" || !result.TraceOn {
+ return err
+ }
+
+ sendSuccess := result.Status == primitive.SendOK
+ costT := time.Since(beginT).Nanoseconds() / int64(time.Millisecond)
+ storeT := beginT.UnixNano()/int64(time.Millisecond) + costT/2
+
+ traceBean := internal.TraceBean{
+ Topic: producerCtx.Message.Topic,
+ Tags: producerCtx.Message.GetTags(),
+ Keys: producerCtx.Message.GetKeys(),
+ StoreHost: producerCtx.BrokerAddr,
+ ClientHost: utils.LocalIP,
+ BodyLength: len(producerCtx.Message.Body),
+ MsgType: producerCtx.MsgType,
+ MsgId: result.MsgID,
+ OffsetMsgId: result.OffsetMsgID,
+ StoreTime: storeT,
+ }
+
+ traceCtx := internal.TraceContext{
+ RequestId: primitive.CreateUniqID(), // set id
+ TimeStamp: time.Now().UnixNano() / int64(time.Millisecond),
+
+ TraceType: internal.Pub,
+ GroupName: producerCtx.ProducerGroup,
+ RegionId: result.RegionID,
+ TraceBeans: []internal.TraceBean{traceBean},
+ CostTime: costT,
+ IsSuccess: sendSuccess,
+ }
+ dispatcher.Append(traceCtx)
+ return err
+ }
+}
diff --git a/producer/option.go b/producer/option.go
new file mode 100644
index 0000000..76e9a31
--- /dev/null
+++ b/producer/option.go
@@ -0,0 +1,144 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package producer
+
+import (
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func defaultProducerOptions() producerOptions {
+ opts := producerOptions{
+ ClientOptions: internal.DefaultClientOptions(),
+ Selector: NewRoundRobinQueueSelector(),
+ SendMsgTimeout: 3 * time.Second,
+ DefaultTopicQueueNums: 4,
+ CreateTopicKey: "TBW102",
+ Resolver: primitive.NewHttpResolver("DEFAULT"),
+ }
+ opts.ClientOptions.GroupName = "DEFAULT_CONSUMER"
+ return opts
+}
+
+type producerOptions struct {
+ internal.ClientOptions
+ Selector QueueSelector
+ SendMsgTimeout time.Duration
+ DefaultTopicQueueNums int
+ CreateTopicKey string // "TBW102" Will be created at broker when isAutoCreateTopicEnable. when topic is not created,
+ // and broker open isAutoCreateTopicEnable, topic will use "TBW102" config to create topic
+ Resolver primitive.NsResolver
+}
+
+type Option func(*producerOptions)
+
+// WithGroupName set group name address
+func WithGroupName(group string) Option {
+ return func(opts *producerOptions) {
+ if group == "" {
+ return
+ }
+ opts.GroupName = group
+ }
+}
+
+func WithInstanceName(name string) Option {
+ return func(opts *producerOptions) {
+ opts.InstanceName = name
+ }
+}
+
+// WithNamespace set the namespace of producer
+func WithNamespace(namespace string) Option {
+ return func(opts *producerOptions) {
+ opts.Namespace = namespace
+ }
+}
+
+func WithSendMsgTimeout(duration time.Duration) Option {
+ return func(opts *producerOptions) {
+ opts.SendMsgTimeout = duration
+ }
+}
+
+func WithVIPChannel(enable bool) Option {
+ return func(opts *producerOptions) {
+ opts.VIPChannelEnabled = enable
+ }
+}
+
+// WithRetry return a Option that specifies the retry times when send failed.
+// TODO: use retry middleware instead
+func WithRetry(retries int) Option {
+ return func(opts *producerOptions) {
+ opts.RetryTimes = retries
+ }
+}
+
+func WithInterceptor(f ...primitive.Interceptor) Option {
+ return func(opts *producerOptions) {
+ opts.Interceptors = append(opts.Interceptors, f...)
+ }
+}
+
+func WithQueueSelector(s QueueSelector) Option {
+ return func(options *producerOptions) {
+ options.Selector = s
+ }
+}
+
+func WithCredentials(c primitive.Credentials) Option {
+ return func(options *producerOptions) {
+ options.ClientOptions.Credentials = c
+ }
+}
+
+func WithDefaultTopicQueueNums(queueNum int) Option {
+ return func(options *producerOptions) {
+ options.DefaultTopicQueueNums = queueNum
+ }
+}
+
+func WithCreateTopicKey(topic string) Option {
+ return func(options *producerOptions) {
+ options.CreateTopicKey = topic
+ }
+}
+
+// WithNsResovler set nameserver resolver to fetch nameserver addr
+func WithNsResovler(resolver primitive.NsResolver) Option {
+ return func(options *producerOptions) {
+ options.Resolver = resolver
+ }
+}
+
+// WithNameServer set NameServer address, only support one NameServer cluster in alpha2
+func WithNameServer(nameServers primitive.NamesrvAddr) Option {
+ return func(options *producerOptions) {
+ options.Resolver = primitive.NewPassthroughResolver(nameServers)
+ }
+}
+
+// WithNameServerDomain set NameServer domain
+func WithNameServerDomain(nameServerUrl string) Option {
+ return func(opts *producerOptions) {
+ opts.Resolver = primitive.NewHttpResolver("DEFAULT", nameServerUrl)
+ }
+}
diff --git a/producer/producer.go b/producer/producer.go
new file mode 100644
index 0000000..7c5e0eb
--- /dev/null
+++ b/producer/producer.go
@@ -0,0 +1,554 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package producer
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/pkg/errors"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/internal/utils"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+ "github.com/apache/rocketmq-client-go/v2/rlog"
+)
+
+var (
+ ErrTopicEmpty = errors.New("topic is nil")
+ ErrMessageEmpty = errors.New("message is nil")
+ ErrNotRunning = errors.New("producer not started")
+)
+
+type defaultProducer struct {
+ group string
+ client internal.RMQClient
+ state int32
+ options producerOptions
+ publishInfo sync.Map
+ callbackCh chan interface{}
+
+ interceptor primitive.Interceptor
+}
+
+func NewDefaultProducer(opts ...Option) (*defaultProducer, error) {
+ defaultOpts := defaultProducerOptions()
+ for _, apply := range opts {
+ apply(&defaultOpts)
+ }
+ srvs, err := internal.NewNamesrv(defaultOpts.Resolver)
+ if err != nil {
+ return nil, errors.Wrap(err, "new Namesrv failed.")
+ }
+ if !defaultOpts.Credentials.IsEmpty() {
+ srvs.SetCredentials(defaultOpts.Credentials)
+ }
+ defaultOpts.Namesrv = srvs
+
+ producer := &defaultProducer{
+ group: defaultOpts.GroupName,
+ callbackCh: make(chan interface{}),
+ options: defaultOpts,
+ }
+ producer.client = internal.GetOrNewRocketMQClient(defaultOpts.ClientOptions, producer.callbackCh)
+
+ producer.interceptor = primitive.ChainInterceptors(producer.options.Interceptors...)
+
+ return producer, nil
+}
+
+func (p *defaultProducer) Start() error {
+ atomic.StoreInt32(&p.state, int32(internal.StateRunning))
+
+ p.client.RegisterProducer(p.group, p)
+ p.client.Start()
+ return nil
+}
+
+func (p *defaultProducer) Shutdown() error {
+ atomic.StoreInt32(&p.state, int32(internal.StateShutdown))
+ p.client.UnregisterProducer(p.group)
+ p.client.Shutdown()
+ return nil
+}
+
+func (p *defaultProducer) checkMsg(msgs ...*primitive.Message) error {
+ if atomic.LoadInt32(&p.state) != int32(internal.StateRunning) {
+ return ErrNotRunning
+ }
+
+ if len(msgs) == 0 {
+ return errors.New("message is nil")
+ }
+
+ if len(msgs[0].Topic) == 0 {
+ return errors.New("topic is nil")
+ }
+ return nil
+}
+
+func (p *defaultProducer) encodeBatch(msgs ...*primitive.Message) *primitive.Message {
+ if len(msgs) == 1 {
+ return msgs[0]
+ }
+
+ // encode batch
+ batch := new(primitive.Message)
+ batch.Topic = msgs[0].Topic
+ batch.Queue = msgs[0].Queue
+ if len(msgs) > 1 {
+ batch.Body = MarshalMessageBatch(msgs...)
+ batch.Batch = true
+ } else {
+ batch.Body = msgs[0].Body
+ batch.Flag = msgs[0].Flag
+ batch.WithProperties(msgs[0].GetProperties())
+ batch.TransactionId = msgs[0].TransactionId
+ }
+ return batch
+}
+
+func MarshalMessageBatch(msgs ...*primitive.Message) []byte {
+ buffer := bytes.NewBufferString("")
+ for _, msg := range msgs {
+ data := msg.Marshal()
+ buffer.Write(data)
+ }
+ return buffer.Bytes()
+}
+
+func (p *defaultProducer) SendSync(ctx context.Context, msgs ...*primitive.Message) (*primitive.SendResult, error) {
+ if err := p.checkMsg(msgs...); err != nil {
+ return nil, err
+ }
+
+ msg := p.encodeBatch(msgs...)
+
+ resp := new(primitive.SendResult)
+ if p.interceptor != nil {
+ primitive.WithMethod(ctx, primitive.SendSync)
+ producerCtx := &primitive.ProducerCtx{
+ ProducerGroup: p.group,
+ CommunicationMode: primitive.SendSync,
+ BornHost: utils.LocalIP,
+ Message: *msg,
+ SendResult: resp,
+ }
+ ctx = primitive.WithProducerCtx(ctx, producerCtx)
+
+ err := p.interceptor(ctx, msg, resp, func(ctx context.Context, req, reply interface{}) error {
+ var err error
+ realReq := req.(*primitive.Message)
+ realReply := reply.(*primitive.SendResult)
+ err = p.sendSync(ctx, realReq, realReply)
+ return err
+ })
+ return resp, err
+ }
+
+ err := p.sendSync(ctx, msg, resp)
+ return resp, err
+}
+
+func (p *defaultProducer) sendSync(ctx context.Context, msg *primitive.Message, resp *primitive.SendResult) error {
+
+ retryTime := 1 + p.options.RetryTimes
+
+ var (
+ err error
+ )
+
+ if p.options.Namespace != "" {
+ msg.Topic = p.options.Namespace + "%" + msg.Topic
+ }
+
+ var producerCtx *primitive.ProducerCtx
+ for retryCount := 0; retryCount < retryTime; retryCount++ {
+ mq := p.selectMessageQueue(msg)
+ if mq == nil {
+ err = fmt.Errorf("the topic=%s route info not found", msg.Topic)
+ continue
+ }
+
+ addr := p.options.Namesrv.FindBrokerAddrByName(mq.BrokerName)
+ if addr == "" {
+ return fmt.Errorf("topic=%s route info not found", mq.Topic)
+ }
+
+ if p.interceptor != nil {
+ producerCtx = primitive.GetProducerCtx(ctx)
+ producerCtx.BrokerAddr = addr
+ producerCtx.MQ = *mq
+ }
+
+ res, _err := p.client.InvokeSync(ctx, addr, p.buildSendRequest(mq, msg), 3*time.Second)
+ if _err != nil {
+ err = _err
+ continue
+ }
+ return p.client.ProcessSendResponse(mq.BrokerName, res, resp, msg)
+ }
+ return err
+}
+
+func (p *defaultProducer) SendAsync(ctx context.Context, f func(context.Context, *primitive.SendResult, error), msgs ...*primitive.Message) error {
+ if err := p.checkMsg(msgs...); err != nil {
+ return err
+ }
+
+ msg := p.encodeBatch(msgs...)
+
+ if p.interceptor != nil {
+ primitive.WithMethod(ctx, primitive.SendAsync)
+
+ return p.interceptor(ctx, msg, nil, func(ctx context.Context, req, reply interface{}) error {
+ return p.sendAsync(ctx, msg, f)
+ })
+ }
+ return p.sendAsync(ctx, msg, f)
+}
+
+func (p *defaultProducer) sendAsync(ctx context.Context, msg *primitive.Message, h func(context.Context, *primitive.SendResult, error)) error {
+ if p.options.Namespace != "" {
+ msg.Topic = p.options.Namespace + "%" + msg.Topic
+ }
+ mq := p.selectMessageQueue(msg)
+ if mq == nil {
+ return errors.Errorf("the topic=%s route info not found", msg.Topic)
+ }
+
+ addr := p.options.Namesrv.FindBrokerAddrByName(mq.BrokerName)
+ if addr == "" {
+ return errors.Errorf("topic=%s route info not found", mq.Topic)
+ }
+
+ ctx, _ = context.WithTimeout(ctx, 3*time.Second)
+ return p.client.InvokeAsync(ctx, addr, p.buildSendRequest(mq, msg), func(command *remote.RemotingCommand, err error) {
+ resp := new(primitive.SendResult)
+ if err != nil {
+ h(ctx, nil, err)
+ } else {
+ p.client.ProcessSendResponse(mq.BrokerName, command, resp, msg)
+ h(ctx, resp, nil)
+ }
+ })
+}
+
+func (p *defaultProducer) SendOneWay(ctx context.Context, msgs ...*primitive.Message) error {
+ if err := p.checkMsg(msgs...); err != nil {
+ return err
+ }
+
+ msg := p.encodeBatch(msgs...)
+
+ if p.interceptor != nil {
+ primitive.WithMethod(ctx, primitive.SendOneway)
+ return p.interceptor(ctx, msg, nil, func(ctx context.Context, req, reply interface{}) error {
+ return p.SendOneWay(ctx, msg)
+ })
+ }
+
+ return p.sendOneWay(ctx, msg)
+}
+
+func (p *defaultProducer) sendOneWay(ctx context.Context, msg *primitive.Message) error {
+ retryTime := 1 + p.options.RetryTimes
+
+ if p.options.Namespace != "" {
+ msg.Topic = p.options.Namespace + "%" + msg.Topic
+ }
+
+ var err error
+ for retryCount := 0; retryCount < retryTime; retryCount++ {
+ mq := p.selectMessageQueue(msg)
+ if mq == nil {
+ err = fmt.Errorf("the topic=%s route info not found", msg.Topic)
+ continue
+ }
+
+ addr := p.options.Namesrv.FindBrokerAddrByName(mq.BrokerName)
+ if addr == "" {
+ return fmt.Errorf("topic=%s route info not found", mq.Topic)
+ }
+
+ _err := p.client.InvokeOneWay(ctx, addr, p.buildSendRequest(mq, msg), 3*time.Second)
+ if _err != nil {
+ err = _err
+ continue
+ }
+ return nil
+ }
+ return err
+}
+
+func (p *defaultProducer) buildSendRequest(mq *primitive.MessageQueue,
+ msg *primitive.Message) *remote.RemotingCommand {
+ if !msg.Batch && msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex) == "" {
+ msg.WithProperty(primitive.PropertyUniqueClientMessageIdKeyIndex, primitive.CreateUniqID())
+ }
+ sysFlag := 0
+ v := msg.GetProperty(primitive.PropertyTransactionPrepared)
+ if v != "" {
+ tranMsg, err := strconv.ParseBool(v)
+ if err == nil && tranMsg {
+ sysFlag |= primitive.TransactionPreparedType
+ }
+ }
+
+ req := &internal.SendMessageRequestHeader{
+ ProducerGroup: p.group,
+ Topic: mq.Topic,
+ QueueId: mq.QueueId,
+ SysFlag: sysFlag,
+ BornTimestamp: time.Now().UnixNano() / int64(time.Millisecond),
+ Flag: msg.Flag,
+ Properties: msg.MarshallProperties(),
+ ReconsumeTimes: 0,
+ UnitMode: p.options.UnitMode,
+ Batch: msg.Batch,
+ }
+ cmd := internal.ReqSendMessage
+ if msg.Batch {
+ cmd = internal.ReqSendBatchMessage
+ reqv2 := &internal.SendMessageRequestV2Header{SendMessageRequestHeader: req}
+ return remote.NewRemotingCommand(cmd, reqv2, msg.Body)
+ }
+
+ return remote.NewRemotingCommand(cmd, req, msg.Body)
+}
+
+func (p *defaultProducer) selectMessageQueue(msg *primitive.Message) *primitive.MessageQueue {
+ topic := msg.Topic
+
+ v, exist := p.publishInfo.Load(topic)
+ if !exist {
+ data, changed, err := p.options.Namesrv.UpdateTopicRouteInfo(topic)
+ if err != nil && primitive.IsRemotingErr(err) {
+ return nil
+ }
+ p.client.UpdatePublishInfo(topic, data, changed)
+ v, exist = p.publishInfo.Load(topic)
+ }
+
+ if !exist {
+ data, changed, _ := p.options.Namesrv.UpdateTopicRouteInfoWithDefault(topic, p.options.CreateTopicKey, p.options.DefaultTopicQueueNums)
+ p.client.UpdatePublishInfo(topic, data, changed)
+ v, exist = p.publishInfo.Load(topic)
+ }
+
+ if !exist {
+ return nil
+ }
+
+ result := v.(*internal.TopicPublishInfo)
+ if result == nil || !result.HaveTopicRouterInfo {
+ return nil
+ }
+
+ if result.MqList != nil && len(result.MqList) <= 0 {
+ rlog.Error("can not find proper message queue", nil)
+ return nil
+ }
+
+ return p.options.Selector.Select(msg, result.MqList)
+}
+
+func (p *defaultProducer) PublishTopicList() []string {
+ topics := make([]string, 0)
+ p.publishInfo.Range(func(key, value interface{}) bool {
+ topics = append(topics, key.(string))
+ return true
+ })
+ return topics
+}
+
+func (p *defaultProducer) UpdateTopicPublishInfo(topic string, info *internal.TopicPublishInfo) {
+ if topic == "" || info == nil {
+ return
+ }
+ p.publishInfo.Store(topic, info)
+}
+
+func (p *defaultProducer) IsPublishTopicNeedUpdate(topic string) bool {
+ v, exist := p.publishInfo.Load(topic)
+ if !exist {
+ return true
+ }
+ info := v.(*internal.TopicPublishInfo)
+ return info.MqList == nil || len(info.MqList) == 0
+}
+
+func (p *defaultProducer) IsUnitMode() bool {
+ return false
+}
+
+type transactionProducer struct {
+ producer *defaultProducer
+ listener primitive.TransactionListener
+}
+
+// TODO: checkLocalTransaction
+func NewTransactionProducer(listener primitive.TransactionListener, opts ...Option) (*transactionProducer, error) {
+ producer, err := NewDefaultProducer(opts...)
+ if err != nil {
+ return nil, errors.Wrap(err, "NewDefaultProducer failed.")
+ }
+ return &transactionProducer{
+ producer: producer,
+ listener: listener,
+ }, nil
+}
+
+func (tp *transactionProducer) Start() error {
+ go primitive.WithRecover(func() {
+ tp.checkTransactionState()
+ })
+ return tp.producer.Start()
+}
+func (tp *transactionProducer) Shutdown() error {
+ return tp.producer.Shutdown()
+}
+
+// TODO: check addr
+func (tp *transactionProducer) checkTransactionState() {
+ for ch := range tp.producer.callbackCh {
+ switch callback := ch.(type) {
+ case *internal.CheckTransactionStateCallback:
+ localTransactionState := tp.listener.CheckLocalTransaction(callback.Msg)
+ uniqueKey := callback.Msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
+ if uniqueKey == "" {
+ uniqueKey = callback.Msg.MsgId
+ }
+ header := &internal.EndTransactionRequestHeader{
+ CommitLogOffset: callback.Header.CommitLogOffset,
+ ProducerGroup: tp.producer.group,
+ TranStateTableOffset: callback.Header.TranStateTableOffset,
+ FromTransactionCheck: true,
+ MsgID: uniqueKey,
+ TransactionId: callback.Header.TransactionId,
+ CommitOrRollback: tp.transactionState(localTransactionState),
+ }
+
+ req := remote.NewRemotingCommand(internal.ReqENDTransaction, header, nil)
+ req.Remark = tp.errRemark(nil)
+
+ err := tp.producer.client.InvokeOneWay(context.Background(), callback.Addr.String(), req,
+ tp.producer.options.SendMsgTimeout)
+ if err != nil {
+ rlog.Error("send ReqENDTransaction to broker error", map[string]interface{}{
+ "callback": callback.Addr.String(),
+ "request": req.String(),
+ rlog.LogKeyUnderlayError: err,
+ })
+ }
+ default:
+ rlog.Error(fmt.Sprintf("unknown type %v", ch), nil)
+ }
+ }
+}
+
+func (tp *transactionProducer) SendMessageInTransaction(ctx context.Context, msg *primitive.Message) (*primitive.TransactionSendResult, error) {
+ msg.WithProperty(primitive.PropertyTransactionPrepared, "true")
+ msg.WithProperty(primitive.PropertyProducerGroup, tp.producer.options.GroupName)
+
+ rsp, err := tp.producer.SendSync(ctx, msg)
+ if err != nil {
+ return nil, err
+ }
+ localTransactionState := primitive.UnknowState
+ switch rsp.Status {
+ case primitive.SendOK:
+ if len(rsp.TransactionID) > 0 {
+ msg.WithProperty("__transactionId__", rsp.TransactionID)
+ }
+ transactionId := msg.GetProperty(primitive.PropertyUniqueClientMessageIdKeyIndex)
+ if len(transactionId) > 0 {
+ msg.TransactionId = transactionId
+ }
+ localTransactionState = tp.listener.ExecuteLocalTransaction(msg)
+ if localTransactionState != primitive.CommitMessageState {
+ rlog.Error("executeLocalTransaction but state unexpected", map[string]interface{}{
+ "localState": localTransactionState,
+ "message": msg,
+ })
+ }
+
+ case primitive.SendFlushDiskTimeout, primitive.SendFlushSlaveTimeout, primitive.SendSlaveNotAvailable:
+ localTransactionState = primitive.RollbackMessageState
+ default:
+ }
+
+ tp.endTransaction(*rsp, err, localTransactionState)
+
+ transactionSendResult := &primitive.TransactionSendResult{
+ SendResult: rsp,
+ State: localTransactionState,
+ }
+
+ return transactionSendResult, nil
+}
+
+func (tp *transactionProducer) endTransaction(result primitive.SendResult, err error, state primitive.LocalTransactionState) error {
+ var msgID *primitive.MessageID
+ if len(result.OffsetMsgID) > 0 {
+ msgID, _ = primitive.UnmarshalMsgID([]byte(result.OffsetMsgID))
+ } else {
+ msgID, _ = primitive.UnmarshalMsgID([]byte(result.MsgID))
+ }
+ // 估计没有反序列化回来
+ brokerAddr := tp.producer.options.Namesrv.FindBrokerAddrByName(result.MessageQueue.BrokerName)
+ requestHeader := &internal.EndTransactionRequestHeader{
+ TransactionId: result.TransactionID,
+ CommitLogOffset: msgID.Offset,
+ ProducerGroup: tp.producer.group,
+ TranStateTableOffset: result.QueueOffset,
+ MsgID: result.MsgID,
+ CommitOrRollback: tp.transactionState(state),
+ }
+
+ req := remote.NewRemotingCommand(internal.ReqENDTransaction, requestHeader, nil)
+ req.Remark = tp.errRemark(err)
+
+ return tp.producer.client.InvokeOneWay(context.Background(), brokerAddr, req, tp.producer.options.SendMsgTimeout)
+}
+
+func (tp *transactionProducer) errRemark(err error) string {
+ if err != nil {
+ return "executeLocalTransactionBranch exception: " + err.Error()
+ }
+ return ""
+}
+
+func (tp *transactionProducer) transactionState(state primitive.LocalTransactionState) int {
+ switch state {
+ case primitive.CommitMessageState:
+ return primitive.TransactionCommitType
+ case primitive.RollbackMessageState:
+ return primitive.TransactionRollbackType
+ case primitive.UnknowState:
+ return primitive.TransactionNotType
+ default:
+ return primitive.TransactionNotType
+ }
+}
diff --git a/producer/producer_test.go b/producer/producer_test.go
new file mode 100644
index 0000000..508acf8
--- /dev/null
+++ b/producer/producer_test.go
@@ -0,0 +1,247 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package producer
+
+import (
+ "context"
+ "testing"
+
+ "github.com/golang/mock/gomock"
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/rocketmq-client-go/v2/internal"
+ "github.com/apache/rocketmq-client-go/v2/internal/remote"
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+const (
+ topic = "TopicTest"
+)
+
+func TestShutdown(t *testing.T) {
+ p, _ := NewDefaultProducer(
+ WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ WithRetry(2),
+ WithQueueSelector(NewManualQueueSelector()),
+ )
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+ client := internal.NewMockRMQClient(ctrl)
+ p.client = client
+
+ client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return()
+ client.EXPECT().Start().Return()
+ err := p.Start()
+ assert.Nil(t, err)
+
+ client.EXPECT().Shutdown().Return()
+ client.EXPECT().UnregisterProducer(gomock.Any()).Return()
+ err = p.Shutdown()
+ assert.Nil(t, err)
+
+ ctx := context.Background()
+ msg := new(primitive.Message)
+
+ r, err := p.SendSync(ctx, msg)
+ assert.Equal(t, ErrNotRunning, err)
+ assert.Nil(t, r)
+
+ err = p.SendOneWay(ctx, msg)
+ assert.Equal(t, ErrNotRunning, err)
+
+ f := func(context.Context, *primitive.SendResult, error) {
+ assert.False(t, true, "should not come in")
+ }
+ err = p.SendAsync(ctx, f, msg)
+ assert.Equal(t, ErrNotRunning, err)
+}
+
+func mockB4Send(p *defaultProducer) {
+ p.publishInfo.Store(topic, &internal.TopicPublishInfo{
+ HaveTopicRouterInfo: true,
+ MqList: []*primitive.MessageQueue{
+ {
+ Topic: topic,
+ BrokerName: "aa",
+ QueueId: 0,
+ },
+ },
+ })
+ p.options.Namesrv.AddBroker(&internal.TopicRouteData{
+ BrokerDataList: []*internal.BrokerData{
+ {
+ Cluster: "cluster",
+ BrokerName: "aa",
+ BrokerAddresses: map[int64]string{
+ 0: "1",
+ },
+ },
+ },
+ })
+}
+
+func TestSync(t *testing.T) {
+ p, _ := NewDefaultProducer(
+ WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ WithRetry(2),
+ WithQueueSelector(NewManualQueueSelector()),
+ )
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+ client := internal.NewMockRMQClient(ctrl)
+ p.client = client
+
+ client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return()
+ client.EXPECT().Start().Return()
+ err := p.Start()
+ assert.Nil(t, err)
+
+ ctx := context.Background()
+ msg := &primitive.Message{
+ Topic: topic,
+ Body: []byte("this is a message body"),
+ Queue: &primitive.MessageQueue{
+ Topic: topic,
+ BrokerName: "aa",
+ QueueId: 0,
+ },
+ }
+ msg.WithProperty("key", "value")
+
+ expectedResp := &primitive.SendResult{
+ Status: primitive.SendOK,
+ MsgID: "111",
+ QueueOffset: 0,
+ OffsetMsgID: "0",
+ }
+
+ mockB4Send(p)
+
+ client.EXPECT().InvokeSync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil)
+ client.EXPECT().ProcessSendResponse(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(
+ func(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) {
+ resp.Status = expectedResp.Status
+ resp.MsgID = expectedResp.MsgID
+ resp.QueueOffset = expectedResp.QueueOffset
+ resp.OffsetMsgID = expectedResp.OffsetMsgID
+ })
+ resp, err := p.SendSync(ctx, msg)
+ assert.Nil(t, err)
+ assert.Equal(t, expectedResp, resp)
+}
+
+func TestASync(t *testing.T) {
+ p, _ := NewDefaultProducer(
+ WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ WithRetry(2),
+ WithQueueSelector(NewManualQueueSelector()),
+ )
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+ client := internal.NewMockRMQClient(ctrl)
+ p.client = client
+
+ client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return()
+ client.EXPECT().Start().Return()
+ err := p.Start()
+ assert.Nil(t, err)
+
+ ctx := context.Background()
+ msg := &primitive.Message{
+ Topic: topic,
+ Body: []byte("this is a message body"),
+ Queue: &primitive.MessageQueue{
+ Topic: topic,
+ BrokerName: "aa",
+ QueueId: 0,
+ },
+ }
+ msg.WithProperty("key", "value")
+
+ expectedResp := &primitive.SendResult{
+ Status: primitive.SendOK,
+ MsgID: "111",
+ QueueOffset: 0,
+ OffsetMsgID: "0",
+ }
+
+ f := func(ctx context.Context, resp *primitive.SendResult, err error) {
+ assert.Nil(t, err)
+ assert.Equal(t, expectedResp, resp)
+ }
+
+ mockB4Send(p)
+
+ client.EXPECT().InvokeAsync(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn(
+ func(ctx context.Context, addr string, request *remote.RemotingCommand,
+ f func(*remote.RemotingCommand, error)) error {
+ // mock invoke callback
+ f(nil, nil)
+ return nil
+ })
+ client.EXPECT().ProcessSendResponse(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Do(
+ func(brokerName string, cmd *remote.RemotingCommand, resp *primitive.SendResult, msgs ...*primitive.Message) {
+ resp.Status = expectedResp.Status
+ resp.MsgID = expectedResp.MsgID
+ resp.QueueOffset = expectedResp.QueueOffset
+ resp.OffsetMsgID = expectedResp.OffsetMsgID
+ })
+
+ err = p.SendAsync(ctx, f, msg)
+ assert.Nil(t, err)
+}
+
+func TestOneway(t *testing.T) {
+ p, _ := NewDefaultProducer(
+ WithNsResovler(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})),
+ WithRetry(2),
+ WithQueueSelector(NewManualQueueSelector()),
+ )
+
+ ctrl := gomock.NewController(t)
+ defer ctrl.Finish()
+ client := internal.NewMockRMQClient(ctrl)
+ p.client = client
+
+ client.EXPECT().RegisterProducer(gomock.Any(), gomock.Any()).Return()
+ client.EXPECT().Start().Return()
+ err := p.Start()
+ assert.Nil(t, err)
+
+ ctx := context.Background()
+ msg := &primitive.Message{
+ Topic: topic,
+ Body: []byte("this is a message body"),
+ Queue: &primitive.MessageQueue{
+ Topic: topic,
+ BrokerName: "aa",
+ QueueId: 0,
+ },
+ }
+ msg.WithProperty("key", "value")
+
+ mockB4Send(p)
+
+ client.EXPECT().InvokeOneWay(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+
+ err = p.SendOneWay(ctx, msg)
+ assert.Nil(t, err)
+}
diff --git a/producer/selector.go b/producer/selector.go
new file mode 100644
index 0000000..1aead8b
--- /dev/null
+++ b/producer/selector.go
@@ -0,0 +1,123 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package producer
+
+import (
+ "hash/fnv"
+ "math/rand"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+type QueueSelector interface {
+ Select(*primitive.Message, []*primitive.MessageQueue) *primitive.MessageQueue
+}
+
+// manualQueueSelector use the queue manually set in the provided Message's QueueID field as the queue to send.
+type manualQueueSelector struct{}
+
+func NewManualQueueSelector() QueueSelector {
+ return new(manualQueueSelector)
+}
+
+func (manualQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
+ return message.Queue
+}
+
+// randomQueueSelector choose a random queue each time.
+type randomQueueSelector struct {
+ rander *rand.Rand
+}
+
+func NewRandomQueueSelector() QueueSelector {
+ s := new(randomQueueSelector)
+ s.rander = rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
+ return s
+}
+
+func (r randomQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
+ i := r.rander.Intn(len(queues))
+ return queues[i]
+}
+
+// roundRobinQueueSelector choose the queue by roundRobin.
+type roundRobinQueueSelector struct {
+ sync.Locker
+ indexer map[string]*int32
+}
+
+func NewRoundRobinQueueSelector() QueueSelector {
+ s := &roundRobinQueueSelector{
+ Locker: new(sync.Mutex),
+ indexer: map[string]*int32{},
+ }
+ return s
+}
+
+func (r *roundRobinQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
+ t := message.Topic
+ if _, exist := r.indexer[t]; !exist {
+ r.Lock()
+ if _, exist := r.indexer[t]; !exist {
+ var v = int32(0)
+ r.indexer[t] = &v
+ }
+ r.Unlock()
+ }
+ index := r.indexer[t]
+
+ i := atomic.AddInt32(index, 1)
+ if i < 0 {
+ i = -i
+ atomic.StoreInt32(index, 0)
+ }
+ qIndex := int(i) % len(queues)
+ return queues[qIndex]
+}
+
+type hashQueueSelector struct {
+ random QueueSelector
+}
+
+func NewHashQueueSelector() QueueSelector {
+ return &hashQueueSelector{
+ random: NewRandomQueueSelector(),
+ }
+}
+
+// hashQueueSelector choose the queue by hash if message having sharding key, otherwise choose queue by random instead.
+func (h *hashQueueSelector) Select(message *primitive.Message, queues []*primitive.MessageQueue) *primitive.MessageQueue {
+ key := message.GetShardingKey()
+ if len(key) == 0 {
+ return h.random.Select(message, queues)
+ }
+
+ hasher := fnv.New32a()
+ _, err := hasher.Write([]byte(key))
+ if err != nil {
+ return nil
+ }
+ queueId := int(hasher.Sum32()) % len(queues)
+ if queueId < 0 {
+ queueId = -queueId
+ }
+ return queues[queueId]
+}
diff --git a/producer/selector_test.go b/producer/selector_test.go
new file mode 100644
index 0000000..72dc469
--- /dev/null
+++ b/producer/selector_test.go
@@ -0,0 +1,78 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package producer
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/apache/rocketmq-client-go/v2/primitive"
+)
+
+func TestRoundRobin(t *testing.T) {
+ queues := make([]*primitive.MessageQueue, 10)
+ for i := 0; i < 10; i++ {
+ queues = append(queues, &primitive.MessageQueue{
+ QueueId: i,
+ })
+ }
+ s := NewRoundRobinQueueSelector()
+
+ m := &primitive.Message{
+ Topic: "test",
+ }
+ mrr := &primitive.Message{
+ Topic: "rr",
+ }
+ for i := 0; i < 100; i++ {
+ q := s.Select(m, queues)
+ expected := (i + 1) % len(queues)
+ assert.Equal(t, queues[expected], q, "i: %d", i)
+
+ qrr := s.Select(mrr, queues)
+ expected = (i + 1) % len(queues)
+ assert.Equal(t, queues[expected], qrr, "i: %d", i)
+ }
+}
+
+func TestHashQueueSelector(t *testing.T) {
+ queues := make([]*primitive.MessageQueue, 10)
+ for i := 0; i < 10; i++ {
+ queues = append(queues, &primitive.MessageQueue{
+ QueueId: i,
+ })
+ }
+
+ s := NewHashQueueSelector()
+
+ m1 := &primitive.Message{
+ Topic: "test",
+ Body: []byte("one message"),
+ }
+ m1.WithShardingKey("same_key")
+ q1 := s.Select(m1, queues)
+
+ m2 := &primitive.Message{
+ Topic: "test",
+ Body: []byte("another message"),
+ }
+ m2.WithShardingKey("same_key")
+ q2 := s.Select(m2, queues)
+ assert.Equal(t, *q1, *q2)
+}
diff --git a/rlog/log.go b/rlog/log.go
new file mode 100644
index 0000000..426f698
--- /dev/null
+++ b/rlog/log.go
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rlog
+
+import (
+ "os"
+ "strings"
+
+ "github.com/sirupsen/logrus"
+)
+
+const (
+ LogKeyConsumerGroup = "consumerGroup"
+ LogKeyTopic = "topic"
+ LogKeyMessageQueue = "MessageQueue"
+ LogKeyUnderlayError = "underlayError"
+ LogKeyBroker = "broker"
+ LogKeyValueChangedFrom = "changedFrom"
+ LogKeyValueChangedTo = "changeTo"
+ LogKeyPullRequest = "PullRequest"
+)
+
+type Logger interface {
+ Debug(msg string, fields map[string]interface{})
+ Info(msg string, fields map[string]interface{})
+ Warning(msg string, fields map[string]interface{})
+ Error(msg string, fields map[string]interface{})
+ Fatal(msg string, fields map[string]interface{})
+}
+
+func init() {
+ r := &defaultLogger{
+ logger: logrus.New(),
+ }
+ level := os.Getenv("ROCKETMQ_GO_LOG_LEVEL")
+ switch strings.ToLower(level) {
+ case "debug":
+ r.logger.SetLevel(logrus.DebugLevel)
+ case "warn":
+ r.logger.SetLevel(logrus.WarnLevel)
+ case "error":
+ r.logger.SetLevel(logrus.ErrorLevel)
+ default:
+ r.logger.SetLevel(logrus.InfoLevel)
+ }
+ rLog = r
+}
+
+var rLog Logger
+
+type defaultLogger struct {
+ logger *logrus.Logger
+}
+
+func (l *defaultLogger) Debug(msg string, fields map[string]interface{}) {
+ if msg == "" && len(fields) == 0 {
+ return
+ }
+ l.logger.WithFields(fields).Debug(msg)
+}
+
+func (l *defaultLogger) Info(msg string, fields map[string]interface{}) {
+ if msg == "" && len(fields) == 0 {
+ return
+ }
+ l.logger.WithFields(fields).Info(msg)
+}
+
+func (l *defaultLogger) Warning(msg string, fields map[string]interface{}) {
+ if msg == "" && len(fields) == 0 {
+ return
+ }
+ l.logger.WithFields(fields).Warning(msg)
+}
+
+func (l *defaultLogger) Error(msg string, fields map[string]interface{}) {
+ if msg == "" && len(fields) == 0 {
+ return
+ }
+ l.logger.WithFields(fields).WithFields(fields).Error(msg)
+}
+
+func (l *defaultLogger) Fatal(msg string, fields map[string]interface{}) {
+ if msg == "" && len(fields) == 0 {
+ return
+ }
+ l.logger.WithFields(fields).Fatal(msg)
+}
+
+// SetLogger use specified logger user customized, in general, we suggest user to replace the default logger with specified
+func SetLogger(logger Logger) {
+ rLog = logger
+}
+
+func Debug(msg string, fields map[string]interface{}) {
+ rLog.Debug(msg, fields)
+}
+
+func Info(msg string, fields map[string]interface{}) {
+ if msg == "" && len(fields) == 0 {
+ return
+ }
+ rLog.Info(msg, fields)
+}
+
+func Warning(msg string, fields map[string]interface{}) {
+ if msg == "" && len(fields) == 0 {
+ return
+ }
+ rLog.Warning(msg, fields)
+}
+
+func Error(msg string, fields map[string]interface{}) {
+ rLog.Error(msg, fields)
+}
+
+func Fatal(msg string, fields map[string]interface{}) {
+ rLog.Fatal(msg, fields)
+}
