[ISSUE #29] Store the broker configuration and jvm parameters in configmap (#25)
* store config in configmap
diff --git a/build/Dockerfile b/build/Dockerfile
index 28ba05a..90ae2d4 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -33,7 +33,7 @@
ENV ROCKETMQ_VERSION 4.5.0
# Rocketmq home
-ENV ROCKETMQ_HOME /home/rocketmq/rocketmq-${ROCKETMQ_VERSION}
+ENV ROCKETMQ_HOME /home/rocketmq/operator
WORKDIR ${ROCKETMQ_HOME}
diff --git a/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml b/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml
index a85bbf4..834c43c 100644
--- a/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml
+++ b/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml
@@ -48,9 +48,6 @@
description: ReplicaPerGroup each broker cluster's replica number
format: int64
type: integer
- replicationMode:
- description: ReplicationMode is SYNC or ASYNC
- type: string
resources:
description: Resources describes the compute resource requirements
type: object
@@ -67,6 +64,16 @@
storageMode:
description: StorageMode can be EmptyDir, HostPath, StorageClass
type: string
+ env:
+ description: env defines custom env, e.g. BROKER_MEM
+ items:
+ type: object
+ type: array
+ volumes:
+ description: volumes defines the broker.conf
+ items:
+ type: object
+ type: array
volumeClaimTemplates:
description: VolumeClaimTemplates defines the StorageClass
items:
@@ -77,10 +84,13 @@
- replicaPerGroup
- brokerImage
- imagePullPolicy
+ - nameServers
- allowRestart
- resources
- storageMode
- hostPath
+ - env
+ - volumes
- volumeClaimTemplates
- scalePodName
type: object
diff --git a/example/rocketmq_v1alpha1_broker_cr.yaml b/example/rocketmq_v1alpha1_broker_cr.yaml
index ff7ec72..f6bea3b 100644
--- a/example/rocketmq_v1alpha1_broker_cr.yaml
+++ b/example/rocketmq_v1alpha1_broker_cr.yaml
@@ -13,6 +13,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: broker-config
+data:
+ BROKER_MEM: " -Xms4g -Xmx4g -Xmn2g "
+ broker-common.conf: |
+ # brokerClusterName, brokerName, brokerId are automatically generated by the operator and do not set it manually!!!
+ deleteWhen=04
+ fileReservedTime=48
+ flushDiskType=ASYNC_FLUSH
+ # If it's a slave/replica, the brokerRole will automatically be set to SLAVE
+ brokerRole=ASYNC_MASTER
+ storePathCommitLog=/home/rocketmq/store/commitlog
+ storePathRootDir=/home/rocketmq/store/
+ storePathScheduledTempData=/home/rocketmq/store/scheduledtempdata
+
+---
apiVersion: rocketmq.apache.org/v1alpha1
kind: Broker
metadata:
@@ -23,8 +41,6 @@
size: 1
# nameServers is the [ip:port] list of name service
nameServers: ""
- # replicationMode is the broker replica sync mode, can be ASYNC or SYNC
- replicationMode: ASYNC
# replicaPerGroup is the number of each broker cluster
replicaPerGroup: 1
# brokerImage is the customized docker image repo of the RocketMQ broker
@@ -45,8 +61,23 @@
storageMode: HostPath
# hostPath is the local path to store data
hostPath: /data/rocketmq/broker
- # scalePodName is broker-[broker group number]-master-0
+ # scalePodName is [Broker name]-[broker group number]-master-0
scalePodName: broker-0-master-0
+ # env defines custom env, e.g. BROKER_MEM
+ env:
+ - name: BROKER_MEM
+ valueFrom:
+ configMapKeyRef:
+ name: broker-config
+ key: BROKER_MEM
+ # volumes defines the broker.conf
+ volumes:
+ - name: broker-config
+ configMap:
+ name: broker-config
+ items:
+ - key: broker-common.conf
+ path: broker-common.conf
# volumeClaimTemplates defines the storageClass
volumeClaimTemplates:
- metadata:
diff --git a/images/broker/alpine/Dockerfile b/images/broker/alpine/Dockerfile
index f25716f..ef36b44 100644
--- a/images/broker/alpine/Dockerfile
+++ b/images/broker/alpine/Dockerfile
@@ -25,7 +25,7 @@
ENV ROCKETMQ_VERSION ${version}
# Rocketmq home
-ENV ROCKETMQ_HOME /root/rocketmq-${ROCKETMQ_VERSION}
+ENV ROCKETMQ_HOME /home/rocketmq/broker
WORKDIR ${ROCKETMQ_HOME}
diff --git a/images/broker/alpine/brokerGenConfig.sh b/images/broker/alpine/brokerGenConfig.sh
index 862af52..e681974 100755
--- a/images/broker/alpine/brokerGenConfig.sh
+++ b/images/broker/alpine/brokerGenConfig.sh
@@ -16,36 +16,21 @@
# limitations under the License.
BROKER_CONFIG_FILE="$ROCKETMQ_HOME/conf/broker.conf"
-echo $BROKER_CONFIG_FILE
-
-BROKER_ROLE="SLAVE"
-
-if [ $BROKER_ID = 0 ];then
- if [ $REPLICATION_MODE = "SYNC" ];then
- BROKER_ROLE="SYNC_MASTER"
- else
- BROKER_ROLE="ASYNC_MASTER"
- fi
-fi
-
-#BROKER_NAME=$(cat /etc/hostname)
-DELETE_WHEN="04"
-FILE_RESERVED_TIME="48"
-FLUSH_DISK_TYPE="ASYNC_FLUSH"
+BROKER_CONFIG_MOUNT_FILE="$ROCKETMQ_HOME/conf/broker-common.conf"
function create_config() {
rm -f $BROKER_CONFIG_FILE
echo "Creating broker configuration."
- echo "#This file was auto generated by rocketmq-operator. DO NOT EDIT." >> $BROKER_CONFIG_FILE
+ # Remove brokerClusterName, brokerName, brokerId if configured
+ sed -e '/brokerClusterName/d;/brokerName/d;/brokerId/d' $BROKER_CONFIG_MOUNT_FILE > $BROKER_CONFIG_FILE
+ echo -e >> $BROKER_CONFIG_FILE
echo "brokerClusterName=$BROKER_CLUSTER_NAME" >> $BROKER_CONFIG_FILE
echo "brokerName=$BROKER_NAME" >> $BROKER_CONFIG_FILE
echo "brokerId=$BROKER_ID" >> $BROKER_CONFIG_FILE
- echo "deleteWhen=$DELETE_WHEN" >> $BROKER_CONFIG_FILE
- echo "fileReservedTime=$FILE_RESERVED_TIME" >> $BROKER_CONFIG_FILE
- echo "brokerRole=$BROKER_ROLE" >> $BROKER_CONFIG_FILE
- echo "flushDiskType=$FLUSH_DISK_TYPE" >> $BROKER_CONFIG_FILE
- echo "Wrote broker configuration file to $BROKER_CONFIG_FILE"
+ if [ $BROKER_ID != 0 ]; then
+ sed -i 's/brokerRole=.*/brokerRole=SLAVE/g' $BROKER_CONFIG_FILE
+ fi
}
-
-create_config
\ No newline at end of file
+create_config
+cat $BROKER_CONFIG_FILE
\ No newline at end of file
diff --git a/images/broker/alpine/runbroker-customize.sh b/images/broker/alpine/runbroker-customize.sh
index e1edad0..842b9d8 100755
--- a/images/broker/alpine/runbroker-customize.sh
+++ b/images/broker/alpine/runbroker-customize.sh
@@ -130,7 +130,7 @@
Xmn=$HEAP_NEWSIZE
MaxDirectMemorySize=$MAX_HEAP_SIZE
# Set for `JAVA_OPT`.
-JAVA_OPT="${JAVA_OPT} -server -Xms${Xms} -Xmx${Xmx} -Xmn${Xmn}"
+JAVA_OPT="${JAVA_OPT} -server ${BROKER_MEM:-"-Xms${Xms} -Xmx${Xmx} -Xmn${Xmn}"} "
JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8"
JAVA_OPT="${JAVA_OPT} -verbose:gc -Xloggc:/dev/shm/mq_gc_%p.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy"
JAVA_OPT="${JAVA_OPT} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m"
diff --git a/images/namesrv/alpine/Dockerfile b/images/namesrv/alpine/Dockerfile
index b0303f2..b105d4a 100644
--- a/images/namesrv/alpine/Dockerfile
+++ b/images/namesrv/alpine/Dockerfile
@@ -25,7 +25,7 @@
ENV ROCKETMQ_VERSION ${version}
# Rocketmq home
-ENV ROCKETMQ_HOME /root/rocketmq-${ROCKETMQ_VERSION}
+ENV ROCKETMQ_HOME /home/rocketmq/nameserver
WORKDIR ${ROCKETMQ_HOME}
diff --git a/pkg/apis/rocketmq/v1alpha1/broker_types.go b/pkg/apis/rocketmq/v1alpha1/broker_types.go
index 8eb31de..1e694ee 100644
--- a/pkg/apis/rocketmq/v1alpha1/broker_types.go
+++ b/pkg/apis/rocketmq/v1alpha1/broker_types.go
@@ -34,8 +34,6 @@
Size int `json:"size"`
// NameServers defines the name service list e.g. 192.168.1.1:9876;192.168.1.2:9876
NameServers string `json:"nameServers,omitempty"`
- // ReplicationMode is SYNC or ASYNC
- ReplicationMode string `json:"replicationMode,omitempty"`
// ReplicaPerGroup each broker cluster's replica number
ReplicaPerGroup int `json:"replicaPerGroup"`
// BaseImage is the broker image to use for the Pods
@@ -50,6 +48,10 @@
StorageMode string `json:"storageMode"`
// HostPath is the local path to store data
HostPath string `json:"hostPath"`
+ // Env defines custom env, e.g. BROKER_MEM
+ Env []corev1.EnvVar `json:"env"`
+ // Volumes define the broker.conf
+ Volumes []corev1.Volume `json:"volumes"`
// VolumeClaimTemplates defines the StorageClass
VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates"`
// The name of pod where the metadata from
diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go
index 233e9c2..bc35a31 100644
--- a/pkg/constants/constants.go
+++ b/pkg/constants/constants.go
@@ -26,7 +26,7 @@
BasicCommand = "sh"
// AdminToolDir is the RocketMQ Admin directory in operator image
- AdminToolDir = "/home/rocketmq/rocketmq-4.5.0/bin/mqadmin"
+ AdminToolDir = "/home/rocketmq/operator/bin/mqadmin"
// StoreConfigDir is the directory of config file
StoreConfigDir = "/home/rocketmq/store/config"
@@ -37,6 +37,12 @@
// SubscriptionGroupJsonDir is the directory of subscriptionGroup.json
SubscriptionGroupJsonDir = "/home/rocketmq/store/config/subscriptionGroup.json"
+ // BrokerConfigDir is the directory of the mounted config file
+ BrokerConfigPath = "/home/rocketmq/broker/conf"
+
+ // BrokerConfigName is the name of mounted configuration file
+ BrokerConfigName = "broker-common.conf"
+
// UpdateBrokerConfig is update broker config command
UpdateBrokerConfig = "updateBrokerConfig"
@@ -46,9 +52,6 @@
// EnvNameServiceAddress is the container environment variable name of name server list
EnvNameServiceAddress = "NAMESRV_ADDR"
- // EnvReplicationMode is the container environment variable name of replication mode
- EnvReplicationMode = "REPLICATION_MODE"
-
// EnvBrokerId is the container environment variable name of broker id
EnvBrokerId = "BROKER_ID"
diff --git a/pkg/controller/broker/broker_controller.go b/pkg/controller/broker/broker_controller.go
index 9055692..73d34dc 100644
--- a/pkg/controller/broker/broker_controller.go
+++ b/pkg/controller/broker/broker_controller.go
@@ -218,7 +218,12 @@
if err != nil {
reqLogger.Error(err, "Failed to get broker replica StatefulSet of "+brokerName)
} else {
- replicaFound.Spec.Template.Spec.Containers[0].Env[0].Value = share.NameServersStr
+ for index := range replicaFound.Spec.Template.Spec.Containers[0].Env {
+ if cons.EnvNameServiceAddress == replicaFound.Spec.Template.Spec.Containers[0].Env[index].Name {
+ replicaFound.Spec.Template.Spec.Containers[0].Env[index].Value = share.NameServersStr
+ break
+ }
+ }
err = r.client.Update(context.TODO(), replicaFound)
if err != nil {
reqLogger.Error(err, "Failed to update NAMESRV_ADDR of "+strconv.Itoa(brokerGroupIndex)+"-replica-"+strconv.Itoa(replicaIndex), "StatefulSet.Namespace", replicaFound.Namespace, "StatefulSet.Name", replicaFound.Name)
@@ -410,22 +415,7 @@
},
},
ImagePullPolicy: broker.Spec.ImagePullPolicy,
- Env: []corev1.EnvVar{{
- Name: cons.EnvNameServiceAddress,
- Value: share.NameServersStr,
- }, {
- Name: cons.EnvReplicationMode,
- Value: broker.Spec.ReplicationMode,
- }, {
- Name: cons.EnvBrokerId,
- Value: strconv.Itoa(replicaIndex),
- }, {
- Name: cons.EnvBrokerClusterName,
- Value: broker.Name,
- }, {
- Name: cons.EnvBrokerName,
- Value: broker.Name + "-" + strconv.Itoa(brokerGroupIndex),
- }},
+ Env: getENV(broker, replicaIndex, brokerGroupIndex),
Ports: []corev1.ContainerPort{{
ContainerPort: cons.BrokerVipContainerPort,
Name: cons.BrokerVipContainerPortName,
@@ -444,6 +434,10 @@
MountPath: cons.StoreMountPath,
Name: broker.Spec.VolumeClaimTemplates[0].Name,
SubPath: cons.StoreSubPathName + getPathSuffix(broker, brokerGroupIndex, replicaIndex),
+ }, {
+ MountPath: cons.BrokerConfigPath + "/" + cons.BrokerConfigName,
+ Name: broker.Spec.Volumes[0].Name,
+ SubPath: cons.BrokerConfigName,
}},
}},
Volumes: getVolumes(broker),
@@ -459,6 +453,24 @@
}
+func getENV(broker *rocketmqv1alpha1.Broker, replicaIndex int, brokerGroupIndex int) []corev1.EnvVar {
+ envs := []corev1.EnvVar{{
+ Name: cons.EnvNameServiceAddress,
+ Value: broker.Spec.NameServers,
+ }, {
+ Name: cons.EnvBrokerId,
+ Value: strconv.Itoa(replicaIndex),
+ }, {
+ Name: cons.EnvBrokerClusterName,
+ Value: broker.Name,
+ }, {
+ Name: cons.EnvBrokerName,
+ Value: broker.Name + "-" + strconv.Itoa(brokerGroupIndex),
+ }}
+ envs = append(envs, broker.Spec.Env...)
+ return envs
+}
+
func getVolumeClaimTemplates(broker *rocketmqv1alpha1.Broker) []corev1.PersistentVolumeClaim {
switch broker.Spec.StorageMode {
case cons.StorageModeStorageClass:
@@ -473,24 +485,26 @@
func getVolumes(broker *rocketmqv1alpha1.Broker) []corev1.Volume {
switch broker.Spec.StorageMode {
case cons.StorageModeStorageClass:
- return nil
+ return broker.Spec.Volumes
case cons.StorageModeEmptyDir:
- volumes := []corev1.Volume{{
+ volumes := broker.Spec.Volumes
+ volumes = append(volumes, corev1.Volume{
Name: broker.Spec.VolumeClaimTemplates[0].Name,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{}},
- }}
+ })
return volumes
case cons.StorageModeHostPath:
fallthrough
default:
- volumes := []corev1.Volume{{
+ volumes := broker.Spec.Volumes
+ volumes = append(volumes, corev1.Volume{
Name: broker.Spec.VolumeClaimTemplates[0].Name,
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: broker.Spec.HostPath,
}},
- }}
+ })
return volumes
}
}
