Adding pinot minion component into helm (#6430)

diff --git a/kubernetes/helm/index.yaml b/kubernetes/helm/index.yaml
index 335fb02..3be9e2a 100644
--- a/kubernetes/helm/index.yaml
+++ b/kubernetes/helm/index.yaml
@@ -2,8 +2,36 @@
 entries:
   pinot:
   - apiVersion: v1
+    appVersion: 0.2.3
+    created: "2021-01-10T20:34:46.225041-08:00"
+    dependencies:
+    - condition: pinot.zookeeper.enabled,zookeeper.enabled
+      name: zookeeper
+      repository: https://charts.helm.sh/incubator
+      version: 2.1.3
+    description: Apache Pinot is a realtime distributed OLAP datastore, which is used
+      to deliver scalable real time analytics with low latency. It can ingest data
+      from offline data sources (such as Hadoop and flat files) as well as online
+      sources (such as Kafka). Pinot is designed to scale horizontally.
+    digest: 5973bae14660ac6b127c6018e142938a0af13a1275960a7ebb1c11987335d0db
+    home: https://pinot.apache.org/
+    keywords:
+    - olap
+    - analytics
+    - database
+    - pinot
+    maintainers:
+    - email: dev@pinot.apache.org
+      name: pinot-dev
+    name: pinot
+    sources:
+    - https://github.com/apache/incubator-pinot/tree/master/kubernetes/helm
+    urls:
+    - pinot-0.2.3.tgz
+    version: 0.2.3
+  - apiVersion: v1
     appVersion: 0.2.2
-    created: "2020-09-29T00:03:26.751495-07:00"
+    created: "2021-01-10T20:34:46.221886-08:00"
     dependencies:
     - condition: pinot.zookeeper.enabled,zookeeper.enabled
       name: zookeeper
@@ -25,13 +53,13 @@
       name: pinot-dev
     name: pinot
     sources:
-    - https://github.com/apache/incubator-pinot
+    - https://github.com/apache/incubator-pinot/tree/master/kubernetes/helm
     urls:
     - pinot-0.2.2.tgz
     version: 0.2.2
   - apiVersion: v1
     appVersion: 0.2.1
-    created: "2020-09-29T00:03:26.748782-07:00"
+    created: "2021-01-10T20:34:46.218993-08:00"
     dependencies:
     - condition: pinot.zookeeper.enabled,zookeeper.enabled
       name: zookeeper
@@ -53,13 +81,13 @@
       name: pinot-dev
     name: pinot
     sources:
-    - https://github.com/apache/incubator-pinot
+    - https://github.com/apache/incubator-pinot/tree/master/kubernetes/helm
     urls:
     - pinot-0.2.1.tgz
     version: 0.2.1
   - apiVersion: v1
     appVersion: 0.2.0
-    created: "2020-09-29T00:03:26.745494-07:00"
+    created: "2021-01-10T20:34:46.213706-08:00"
     dependencies:
     - condition: pinot.zookeeper.enabled,zookeeper.enabled
       name: zookeeper
@@ -81,14 +109,14 @@
       name: pinot-dev
     name: pinot
     sources:
-    - https://github.com/apache/incubator-pinot
+    - https://github.com/apache/incubator-pinot/tree/master/kubernetes/helm
     urls:
     - pinot-0.2.0.tgz
     version: 0.2.0
   presto:
   - apiVersion: v1
     appVersion: 0.2.0
-    created: "2020-09-29T00:03:26.753701-07:00"
+    created: "2021-01-10T20:34:46.22658-08:00"
     description: Presto is an open source distributed SQL query engine for running
       interactive analytic queries against data sources of all sizes ranging from
       gigabytes to petabytes.
@@ -107,4 +135,4 @@
     urls:
     - presto-0.2.0.tgz
     version: 0.2.0
-generated: "2020-09-29T00:03:26.739572-07:00"
+generated: "2021-01-10T20:34:46.210113-08:00"
diff --git a/kubernetes/helm/pinot-0.2.3.tgz b/kubernetes/helm/pinot-0.2.3.tgz
new file mode 100644
index 0000000..67166ce
--- /dev/null
+++ b/kubernetes/helm/pinot-0.2.3.tgz
Binary files differ
diff --git a/kubernetes/helm/pinot/Chart.yaml b/kubernetes/helm/pinot/Chart.yaml
index 9b7f130..f04b664 100644
--- a/kubernetes/helm/pinot/Chart.yaml
+++ b/kubernetes/helm/pinot/Chart.yaml
@@ -18,10 +18,10 @@
 #
 
 apiVersion: v1
-appVersion: 0.2.2
+appVersion: 0.2.3
 name: pinot
 description: Apache Pinot is a realtime distributed OLAP datastore, which is used to deliver scalable real time analytics with low latency. It can ingest data from offline data sources (such as Hadoop and flat files) as well as online sources (such as Kafka). Pinot is designed to scale horizontally.
-version: 0.2.2
+version: 0.2.3
 keywords:
   - olap
   - analytics
diff --git a/kubernetes/helm/pinot/README.md b/kubernetes/helm/pinot/README.md
index e5a7ebf..4af9b94 100644
--- a/kubernetes/helm/pinot/README.md
+++ b/kubernetes/helm/pinot/README.md
@@ -365,6 +365,27 @@
 | `server.updateStrategy.type`                   | StatefulSet update strategy to use.                                                                                                                                        | `RollingUpdate`                                                    |
 | `server.extra.configs`                         | Extra configs append to 'pinot-server.conf' file to start Pinot Server                                                                                                     | `pinot.set.instance.id.to.hostname=true`                           |
 |------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------|
+| `minion.name`                                  | Name of Pinot Minion                                                                                                                                                       | `minion`                                                           |
+| `minion.port`                                  | Pinot minion netty port                                                                                                                                                    | `9514`                                                             |
+| `minion.replicaCount`                          | Pinot minion replicas                                                                                                                                                      | `1`                                                                |
+| `minion.dataDir`                               | Pinot minion data directory, should be same as `minion.persistence.mountPath` or a sub directory of it                                                                     | `/var/pinot/minion/data`                                           |
+| `minion.persistence.enabled`                   | Use a PVC to persist Pinot minion data                                                                                                                                     | `true`                                                             |
+| `minion.persistence.accessMode`                | Access mode of data volume                                                                                                                                                 | `ReadWriteOnce`                                                    |
+| `minion.persistence.size`                      | Size of data volume                                                                                                                                                        | `4G`                                                               |
+| `minion.persistence.mountPath`                 | Mount path of minion data volume                                                                                                                                           | `/var/pinot/minion/data`                                           |
+| `minion.persistence.storageClass`              | Storage class of backing PVC                                                                                                                                               | `""`                                                               |
+| `minion.jvmOpts`                               | Pinot minion JVM Options                                                                                                                                                   | `-Xms512M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -Xloggc:/opt/pinot/gc-pinot-minion.log` |
+| `minion.log4j2ConfFile`                        | Pinot minion log4j2 configuration file                                                                                                                                     | `/opt/pinot/conf/pinot-minion-log4j2.xml`                          |
+| `minion.pluginsDir`                            | Pinot minion plugins directory                                                                                                                                             | `/opt/pinot/plugins`                                               |
+| `minion.service.port`                          | Service Port                                                                                                                                                               | `9514`                                                             |
+| `minion.resources`                             | Pinot minion resource requests and limits                                                                                                                                  | `{}`                                                               |
+| `minion.nodeSelector`                          | Node labels for minion pod assignment                                                                                                                                      | `{}`                                                               |
+| `minion.affinity`                              | Defines affinities and anti-affinities for pods as defined in: <https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity> preferences | `{}`                                                               |
+| `minion.tolerations`                           | List of node tolerations for the pods. <https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/>                                                           | `[]`                                                               |
+| `minion.podAnnotations`                        | Annotations to be added to minion pod                                                                                                                                      | `{}`                                                               |
+| `minion.updateStrategy.type`                   | StatefulSet update strategy to use.                                                                                                                                        | `RollingUpdate`                                                    |
+| `minion.extra.configs`                         | Extra configs append to 'pinot-minion.conf' file to start Pinot Minion                                                                                                     | `pinot.set.instance.id.to.hostname=true`                           |
+|------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------|
 | `zookeeper.enabled`                            | If True, installs Zookeeper Chart                                                                                                                                          | `true`                                                             |
 | `zookeeper.resources`                          | Zookeeper resource requests and limits                                                                                                                                     | `{}`                                                               |
 | `zookeeper.env`                                | Environmental variables provided to Zookeeper Zookeeper                                                                                                                    | `{ZK_HEAP_SIZE: "256M"}`                                           |
diff --git a/kubernetes/helm/pinot/templates/_helpers.tpl b/kubernetes/helm/pinot/templates/_helpers.tpl
index 74c50e9..fcfd229 100644
--- a/kubernetes/helm/pinot/templates/_helpers.tpl
+++ b/kubernetes/helm/pinot/templates/_helpers.tpl
@@ -107,6 +107,15 @@
 {{ template "pinot.fullname" . }}-{{ .Values.server.name }}
 {{- end -}}
 
+
+{{/*
+Create a default fully qualified pinot minion name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "pinot.minion.fullname" -}}
+{{ template "pinot.fullname" . }}-{{ .Values.minion.name }}
+{{- end -}}
+
 {{/*
 The name of the pinot controller headless service.
 */}}
@@ -129,6 +138,13 @@
 {{- end -}}
 
 {{/*
+The name of the pinot minion headless service.
+*/}}
+{{- define "pinot.minion.headless" -}}
+{{- printf "%s-headless" (include "pinot.minion.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
 The name of the pinot controller external service.
 */}}
 {{- define "pinot.controller.external" -}}
@@ -162,3 +178,10 @@
 {{- define "pinot.server.config" -}}
 {{- printf "%s-config" (include "pinot.server.fullname" .) | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{/*
+The name of the pinot minion config.
+*/}}
+{{- define "pinot.minion.config" -}}
+{{- printf "%s-config" (include "pinot.minion.fullname" .) | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/kubernetes/helm/pinot/templates/minion/configmap.yaml b/kubernetes/helm/pinot/templates/minion/configmap.yaml
new file mode 100644
index 0000000..137839a
--- /dev/null
+++ b/kubernetes/helm/pinot/templates/minion/configmap.yaml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "pinot.minion.config" . }}
+data:
+  pinot-minion.conf: |-
+    pinot.minion.port={{ .Values.minion.port }}
+    {{- if .Values.minion.dataDir }}
+    dataDir={{ .Values.minion.dataDir }}
+    {{- end }}
+{{ .Values.minion.extra.configs | indent 4 }}
diff --git a/kubernetes/helm/pinot/templates/minion/service-headless.yaml b/kubernetes/helm/pinot/templates/minion/service-headless.yaml
new file mode 100644
index 0000000..84dae8e
--- /dev/null
+++ b/kubernetes/helm/pinot/templates/minion/service-headless.yaml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "pinot.minion.headless" . }}
+  labels:
+    app: {{ include "pinot.name" . }}
+    chart: {{ include "pinot.chart" . }}
+    component: {{ .Values.minion.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  clusterIP: None
+  ports:
+    # [pod_name].[service_name].[namespace].svc.cluster.local
+    - port: {{ .Values.minion.service.port }}
+  selector:
+    app: {{ include "pinot.name" . }}
+    release: {{ .Release.Name }}
+    component: {{ .Values.minion.name }}
diff --git a/kubernetes/helm/pinot/templates/minion/service.yaml b/kubernetes/helm/pinot/templates/minion/service.yaml
new file mode 100644
index 0000000..cf60983
--- /dev/null
+++ b/kubernetes/helm/pinot/templates/minion/service.yaml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "pinot.minion.fullname" . }}
+  labels:
+    app: {{ include "pinot.name" . }}
+    chart: {{ include "pinot.chart" . }}
+    component: {{ .Values.minion.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: ClusterIP
+  ports:
+    # [pod_name].[service_name].[namespace].svc.cluster.local
+    - port: {{ .Values.minion.service.port }}
+  selector:
+    app: {{ include "pinot.name" . }}
+    release: {{ .Release.Name }}
+    component: {{ .Values.minion.name }}
diff --git a/kubernetes/helm/pinot/templates/minion/statefulset.yml b/kubernetes/helm/pinot/templates/minion/statefulset.yml
new file mode 100644
index 0000000..959203e
--- /dev/null
+++ b/kubernetes/helm/pinot/templates/minion/statefulset.yml
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "pinot.minion.fullname" . }}
+  labels:
+    app: {{ include "pinot.name" . }}
+    chart: {{ include "pinot.chart" . }}
+    component: {{ .Values.minion.name }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ include "pinot.name" . }}
+      release: {{ .Release.Name }}
+      component: {{ .Values.minion.name }}
+  serviceName: {{ template "pinot.minion.headless" . }}
+  replicas: {{ .Values.minion.replicaCount }}
+  updateStrategy:
+    type: {{ .Values.minion.updateStrategy.type }}
+  podManagementPolicy: Parallel
+  template:
+    metadata:
+      labels:
+        app: {{ include "pinot.name" . }}
+        release: {{ .Release.Name }}
+        component: {{ .Values.minion.name }}
+      annotations:
+{{ toYaml .Values.minion.podAnnotations | indent 8 }}
+    spec:
+      nodeSelector:
+{{ toYaml .Values.minion.nodeSelector | indent 8 }}
+      affinity:
+{{ toYaml .Values.minion.affinity | indent 8 }}
+      tolerations:
+{{ toYaml .Values.minion.tolerations | indent 8 }}
+      containers:
+      - name: minion
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        args: [
+          "StartMinion",
+          "-clusterName", "{{ .Values.cluster.name }}",
+          "-zkAddress", {{ include "zookeeper.url" . | quote }},
+          "-configFileName", "/var/pinot/minion/config/pinot-minion.conf"
+        ]
+        env:
+          - name: JAVA_OPTS
+            value: "{{ .Values.minion.jvmOpts }} -Dlog4j2.configurationFile={{ .Values.minion.log4j2ConfFile }} -Dplugins.dir={{ .Values.minion.pluginsDir }}"
+        ports:
+          - containerPort: {{ .Values.minion.port }}
+            protocol: TCP
+        volumeMounts:
+          - name: config
+            mountPath: /var/pinot/minion/config
+          {{- if .Values.minion.persistence.enabled }}
+          - name: data
+            mountPath: "{{ .Values.minion.persistence.mountPath }}"
+          {{- end }}
+        resources:
+{{ toYaml .Values.minion.resources | indent 12 }}
+      restartPolicy: Always
+      volumes:
+        - name: config
+          configMap:
+            name: {{ include "pinot.minion.config" . }}
+      {{- if not .Values.minion.persistence.enabled }}
+        - name: data
+          emptyDir: {}
+      {{- end }}
+
+  {{- if .Values.minion.persistence.enabled }}
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes:
+          - {{ .Values.minion.persistence.accessMode | quote }}
+        {{- if .Values.minion.persistence.storageClass }}
+        {{- if (eq "-" .Values.minion.persistence.storageClass) }}
+        storageClassName: ""
+        {{- else }}
+        storageClassName: {{ .Values.minion.persistence.storageClass }}
+        {{- end }}
+        {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.minion.persistence.size }}
+  {{ end }}
diff --git a/kubernetes/helm/pinot/values.yaml b/kubernetes/helm/pinot/values.yaml
index c36009e..8636a03 100644
--- a/kubernetes/helm/pinot/values.yaml
+++ b/kubernetes/helm/pinot/values.yaml
@@ -187,6 +187,56 @@
     configs: |-
       pinot.set.instance.id.to.hostname=true
       pinot.server.instance.realtime.alloc.offheap=true
+
+minion:
+  name: minion
+
+  port: 9514
+
+  replicaCount: 1
+
+  dataDir: /var/pinot/minion/data
+  jvmOpts: "-Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -Xloggc:/opt/pinot/gc-pinot-minion.log"
+
+  log4j2ConfFile: /opt/pinot/conf/pinot-minion-log4j2.xml
+  pluginsDir: /opt/pinot/plugins
+
+  persistence:
+    enabled: true
+    accessMode: ReadWriteOnce
+    size: 4G
+    mountPath: /var/pinot/minion/data
+    storageClass: ""
+    #storageClass: "ssd"
+
+  service:
+    annotations: {}
+    clusterIP: ""
+    externalIPs: []
+    loadBalancerIP: ""
+    loadBalancerSourceRanges: []
+    type: ClusterIP
+    port: 9514
+    nodePort: ""
+
+  resources: {}
+
+  nodeSelector: {}
+
+  affinity: {}
+
+  tolerations: []
+
+  podAnnotations: {}
+
+  updateStrategy:
+    type: RollingUpdate
+
+  # Extra configs will be appended to pinot-minion.conf file
+  extra:
+    configs: |-
+      pinot.set.instance.id.to.hostname=true
+
 # ------------------------------------------------------------------------------
 # Zookeeper:
 # ------------------------------------------------------------------------------
diff --git a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
index c8d4883..0d3d40f 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionStarter.java
@@ -74,9 +74,12 @@
   public MinionStarter(String helixClusterName, String zkAddress, PinotConfiguration config)
       throws Exception {
     _config = config;
-    _instanceId = config.getProperty(CommonConstants.Helix.Instance.INSTANCE_ID_KEY,
-        CommonConstants.Helix.PREFIX_OF_MINION_INSTANCE + NetUtil.getHostAddress() + "_" + _config
-            .getProperty(CommonConstants.Helix.KEY_OF_MINION_PORT, CommonConstants.Minion.DEFAULT_HELIX_PORT));
+    String host = _config.getProperty(CommonConstants.Helix.KEY_OF_SERVER_NETTY_HOST,
+        _config.getProperty(CommonConstants.Helix.SET_INSTANCE_ID_TO_HOSTNAME_KEY, false) ? NetUtil
+            .getHostnameOrAddress() : NetUtil.getHostAddress());
+    int port = _config
+        .getProperty(CommonConstants.Helix.KEY_OF_MINION_PORT, CommonConstants.Minion.DEFAULT_HELIX_PORT);
+    _instanceId = _config.getProperty(CommonConstants.Helix.Instance.INSTANCE_ID_KEY, CommonConstants.Helix.PREFIX_OF_MINION_INSTANCE + host + "_" + port);
     setupHelixSystemProperties();
     _helixManager = new ZKHelixManager(helixClusterName, _instanceId, InstanceType.PARTICIPANT, zkAddress);
     MinionTaskZkMetadataManager minionTaskZkMetadataManager = new MinionTaskZkMetadataManager(_helixManager);
@@ -140,9 +143,10 @@
     File dataDir = new File(_config
         .getProperty(CommonConstants.Helix.Instance.DATA_DIR_KEY, CommonConstants.Minion.DEFAULT_INSTANCE_DATA_DIR));
     if (dataDir.exists()) {
-      FileUtils.forceDelete(dataDir);
+      FileUtils.cleanDirectory(dataDir);
+    } else {
+      FileUtils.forceMkdir(dataDir);
     }
-    FileUtils.forceMkdir(dataDir);
     minionContext.setDataDir(dataDir);
 
     // Initialize metrics
diff --git a/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java b/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
index 6eb4968..535248f 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/executor/SegmentGenerationAndPushTaskExecutor.java
@@ -97,7 +97,7 @@
   private static final long DEFAULT_PUSH_RETRY_INTERVAL_MILLIS = 1000L;
 
   @Override
-  public Object executeTask(PinotTaskConfig pinotTaskConfig) {
+  public Object executeTask(PinotTaskConfig pinotTaskConfig) throws Exception {
     LOGGER.info("Executing SegmentGenerationAndPushTask with task config: {}", pinotTaskConfig);
     Map<String, String> taskConfigs = pinotTaskConfig.getConfigs();
     SegmentGenerationAndPushResult.Builder resultBuilder = new SegmentGenerationAndPushResult.Builder();
@@ -122,7 +122,7 @@
           outputSegmentTarURI);
       resultBuilder.setSucceed(true);
     } catch (Exception e) {
-      resultBuilder.setException(e);
+      throw new RuntimeException("Failed to execute SegmentGenerationAndPushTask", e);
     } finally {
       // Cleanup output dir
       FileUtils.deleteQuietly(localTempDir);
diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java
index 74cb81f..bc79131 100644
--- a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java
+++ b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/StartMinionCommand.java
@@ -52,7 +52,6 @@
   private String _clusterName = "PinotCluster";
   @Option(name = "-configFileName", required = false, metaVar = "<Config File Name>", usage = "Minion Starter Config file.", forbids = {"-minionHost", "-minionPort"})
   private String _configFileName;
-  private MinionStarter _minionStarter;
 
   public boolean getHelp() {
     return _help;
@@ -74,9 +73,6 @@
 
   @Override
   public void cleanup() {
-    if (_minionStarter != null) {
-      _minionStarter.stop();
-    }
   }
 
   @Override
@@ -130,4 +126,8 @@
     _clusterName = clusterName;
     return this;
   }
+
+  public void setConfigFileName(String configFileName) {
+    _configFileName = configFileName;
+  }
 }
diff --git a/pinot-tools/src/main/resources/conf/pinot-minion-log4j2.xml b/pinot-tools/src/main/resources/conf/pinot-minion-log4j2.xml
new file mode 100644
index 0000000..b8d272b
--- /dev/null
+++ b/pinot-tools/src/main/resources/conf/pinot-minion-log4j2.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<Configuration>
+  <Appenders>
+    <Console name="console" target="SYSTEM_OUT">
+      <PatternLayout>
+        <pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</pattern>
+      </PatternLayout>
+    </Console>
+    <RandomAccessFile name="minionLog" fileName="pinotMinion.log" immediateFlush="false">
+      <PatternLayout>
+        <Pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Pattern>
+      </PatternLayout>
+    </RandomAccessFile>
+  </Appenders>
+  <Loggers>
+    <Root level="info" additivity="false">
+      <!-- Display warnings on the console -->
+      <AppenderRef ref="console" level="warn"/>
+      <!-- Direct most logs to the log file -->
+      <AppenderRef ref="minionLog"/>
+    </Root>
+    <!-- Output server starter logs to the console -->
+    <Logger name="org.apache.pinot.minion.MinionStarter" level="info" additivity="false">
+      <AppenderRef ref="console"/>
+    </Logger>
+    <AsyncLogger name="org.reflections" level="error" additivity="false"/>
+  </Loggers>
+</Configuration>
diff --git a/pinot-tools/src/main/resources/conf/pinot-service-log4j2.xml b/pinot-tools/src/main/resources/conf/pinot-service-log4j2.xml
index f9f3c1e..966625e 100644
--- a/pinot-tools/src/main/resources/conf/pinot-service-log4j2.xml
+++ b/pinot-tools/src/main/resources/conf/pinot-service-log4j2.xml
@@ -42,6 +42,11 @@
         <Pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Pattern>
       </PatternLayout>
     </RandomAccessFile>
+    <RandomAccessFile name="minionLog" fileName="pinotMinion.log" immediateFlush="false">
+      <PatternLayout>
+        <Pattern>%d{yyyy/MM/dd HH:mm:ss.SSS} %p [%c{1}] [%t] %m%n</Pattern>
+      </PatternLayout>
+    </RandomAccessFile>
 
   </Appenders>
   <Loggers>
@@ -79,6 +84,9 @@
     <AsyncLogger name="org.apache.pinot.core.query" level="info" additivity="false">
       <AppenderRef ref="serverLog"/>
     </AsyncLogger>
+    <AsyncLogger name="org.apache.pinot.minion" level="info" additivity="false">
+      <AppenderRef ref="minionLog"/>
+    </AsyncLogger>
     <AsyncLogger name="org.reflections" level="error" additivity="false"/>
     <AsyncLogger name="org.apache.pinot.spi.plugin" level="error" additivity="false">
       <AppenderRef ref="console"/>