examples/kubernetes_tailfile_config.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

MiNiFi Config Version: 3
Flow Controller:
  name: Read Kubernetes pod logs and publish them to Kafka
Processors:
- name: Tail Kubernetes log files
  id: 891efd7f-2814-4068-9efd-5f258f4990d5
  class: org.apache.nifi.minifi.processors.TailFile
  scheduling strategy: TIMER_DRIVEN
  scheduling period: 1 sec
  Properties:
    tail-mode: Multiple file
    tail-base-directory: /var/log/pods/${namespace}_${pod}_${uid}/${container}
    File to Tail: '.*\.log'
    Attribute Provider Service: KubernetesControllerService
    Lookup frequency: 10 min
    Recursive lookup: 'false'
    Initial Start Position: Beginning of File
    Input Delimiter: \n
    Rolling Filename Pattern: '${filename}.log.*'
- name: Publish messages to Kafka topic test
  id: fb880b73-bff7-4775-a854-e048ae09e07e
  class: org.apache.nifi.processors.standard.PublishKafka
  scheduling strategy: EVENT_DRIVEN
  auto-terminated relationships list:
  - success
  - failure
  Properties:
    Batch Size: '10'
    Client Name: test-client
    Compress Codec: none
    Delivery Guarantee: '1'
    Known Brokers: kafka-broker:9092
    Message Timeout: 12 sec
    Request Timeout: 10 sec
    Topic Name: test
Connections:
- name: TailFile/success/PublishKafka
  id: d6675f90-62a6-4f98-b67f-00efeab78e5e
  source id: 891efd7f-2814-4068-9efd-5f258f4990d5
  source relationship name: success
  destination id: fb880b73-bff7-4775-a854-e048ae09e07e
Controller Services:
- name: KubernetesControllerService
  id: 568559dc-3c81-4b01-bc08-760fadb953b0
  type: org.apache.nifi.minifi.controllers.KubernetesControllerService
  Properties:
    Namespace Filter: default
Remote Process Groups: []