| # This ConfigMap is used to ingest logs against new resources like |
| # "k8s_container" and "k8s_node" when $LOGGING_STACKDRIVER_RESOURCE_TYPES is set |
| # to "new". |
| # When $LOGGING_STACKDRIVER_RESOURCE_TYPES is set to "old", the ConfigMap in |
| # fluentd-gcp-configmap-old.yaml will be used for ingesting logs against old |
| # resources like "gke_container" and "gce_instance". |
| kind: ConfigMap |
| apiVersion: v1 |
| data: |
| containers.input.conf: |- |
| # This configuration file for Fluentd is used |
| # to watch changes to Docker log files that live in the |
| # directory /var/lib/docker/containers/ and are symbolically |
| # linked to from the /var/log/containers directory using names that capture the |
| # pod name and container name. These logs are then submitted to |
| # Google Cloud Logging which assumes the installation of the cloud-logging plug-in. |
| # |
| # Example |
| # ======= |
| # A line in the Docker log file might look like this JSON: |
| # |
| # {"log":"2014/09/25 21:15:03 Got request with path wombat\\n", |
| # "stream":"stderr", |
| # "time":"2014-09-25T21:15:03.499185026Z"} |
| # |
| # The original tag is derived from the log file's location. |
| # For example a Docker container's logs might be in the directory: |
| # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b |
| # and in the file: |
| # 997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log |
| # where 997599971ee6... is the Docker ID of the running container. |
| # The Kubernetes kubelet makes a symbolic link to this file on the host |
| # machine in the /var/log/containers directory which includes the pod name, |
| # the namespace name and the Kubernetes container name: |
| # synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log |
| # -> |
| # /var/lib/docker/containers/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b/997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b-json.log |
| # The /var/log directory on the host is mapped to the /var/log directory in the container |
| # running this instance of Fluentd and we end up collecting the file: |
| # /var/log/containers/synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log |
| # This results in the tag: |
| # var.log.containers.synthetic-logger-0.25lps-pod_default_synth-lgr-997599971ee6366d4a5920d25b79286ad45ff37a74494f262e3bc98d909d0a7b.log |
| # where 'synthetic-logger-0.25lps-pod' is the pod name, 'default' is the |
| # namespace name, 'synth-lgr' is the container name and '997599971ee6..' is |
| # the container ID. |
| # The record reformer is used to extract pod_name, namespace_name and |
| # container_name from the tag and set them in a local_resource_id in the |
| # format of: |
| # 'k8s_container.<NAMESPACE_NAME>.<POD_NAME>.<CONTAINER_NAME>'. |
| # The reformer also changes the tags to 'stderr' or 'stdout' based on the |
| # value of 'stream'. |
| # local_resource_id is later used by google_cloud plugin to determine the |
| # monitored resource to ingest logs against. |
| |
| # Json Log Example: |
| # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"} |
| # CRI Log Example: |
| # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here |
| <source> |
| @type tail |
| path /var/log/containers/*.log |
| pos_file /var/log/gcp-containers.log.pos |
| # Tags at this point are in the format of: |
| # reform.var.log.containers.<POD_NAME>_<NAMESPACE_NAME>_<CONTAINER_NAME>-<CONTAINER_ID>.log |
| tag reform.* |
| read_from_head true |
| format multi_format |
| <pattern> |
| format json |
| time_key time |
| time_format %Y-%m-%dT%H:%M:%S.%NZ |
| </pattern> |
| <pattern> |
| format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/ |
| time_format %Y-%m-%dT%H:%M:%S.%N%:z |
| </pattern> |
| </source> |
| |
| <filter reform.**> |
| @type parser |
| format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<log>.*)/ |
| reserve_data true |
| suppress_parse_error_log true |
| emit_invalid_record_to_error false |
| key_name log |
| </filter> |
| |
| <match reform.**> |
| @type record_reformer |
| enable_ruby true |
| <record> |
| # Extract local_resource_id from tag for 'k8s_container' monitored |
| # resource. The format is: |
| # 'k8s_container.<namespace_name>.<pod_name>.<container_name>'. |
| "logging.googleapis.com/local_resource_id" ${"k8s_container.#{tag_suffix[4].rpartition('.')[0].split('_')[1]}.#{tag_suffix[4].rpartition('.')[0].split('_')[0]}.#{tag_suffix[4].rpartition('.')[0].split('_')[2].rpartition('-')[0]}"} |
| # Rename the field 'log' to a more generic field 'message'. This way the |
| # fluent-plugin-google-cloud knows to flatten the field as textPayload |
| # instead of jsonPayload after extracting 'time', 'severity' and |
| # 'stream' from the record. |
| message ${record['log']} |
| # If 'severity' is not set, assume stderr is ERROR and stdout is INFO. |
| severity ${record['severity'] || if record['stream'] == 'stderr' then 'ERROR' else 'INFO' end} |
| </record> |
| tag ${if record['stream'] == 'stderr' then 'raw.stderr' else 'raw.stdout' end} |
| remove_keys stream,log |
| </match> |
| |
| # Detect exceptions in the log output and forward them as one log entry. |
| <match {raw.stderr,raw.stdout}> |
| @type detect_exceptions |
| |
| remove_tag_prefix raw |
| message message |
| stream "logging.googleapis.com/local_resource_id" |
| multiline_flush_interval 5 |
| max_bytes 500000 |
| max_lines 1000 |
| </match> |
| system.input.conf: |- |
| # Example: |
| # Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script |
| <source> |
| @type tail |
| format syslog |
| path /var/log/startupscript.log |
| pos_file /var/log/gcp-startupscript.log.pos |
| tag startupscript |
| </source> |
| |
| # Examples: |
| # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json" |
| # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404 |
| # TODO(random-liu): Remove this after cri container runtime rolls out. |
| <source> |
| @type tail |
| format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ |
| path /var/log/docker.log |
| pos_file /var/log/gcp-docker.log.pos |
| tag docker |
| </source> |
| |
| # Example: |
| # 2016/02/04 06:52:38 filePurge: successfully removed file /var/etcd/data/member/wal/00000000000006d0-00000000010a23d1.wal |
| <source> |
| @type tail |
| # Not parsing this, because it doesn't have anything particularly useful to |
| # parse out of it (like severities). |
| format none |
| path /var/log/etcd.log |
| pos_file /var/log/gcp-etcd.log.pos |
| tag etcd |
| </source> |
| |
| # Multi-line parsing is required for all the kube logs because very large log |
| # statements, such as those that include entire object bodies, get split into |
| # multiple lines by glog. |
| |
| # Example: |
| # I0204 07:32:30.020537 3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537] |
| <source> |
| @type tail |
| format multiline |
| multiline_flush_interval 5s |
| format_firstline /^\w\d{4}/ |
| format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ |
| time_format %m%d %H:%M:%S.%N |
| path /var/log/kubelet.log |
| pos_file /var/log/gcp-kubelet.log.pos |
| tag kubelet |
| </source> |
| |
| # Example: |
| # I1118 21:26:53.975789 6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed |
| <source> |
| @type tail |
| format multiline |
| multiline_flush_interval 5s |
| format_firstline /^\w\d{4}/ |
| format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ |
| time_format %m%d %H:%M:%S.%N |
| path /var/log/kube-proxy.log |
| pos_file /var/log/gcp-kube-proxy.log.pos |
| tag kube-proxy |
| </source> |
| |
| # Example: |
| # I0204 07:00:19.604280 5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266] |
| <source> |
| @type tail |
| format multiline |
| multiline_flush_interval 5s |
| format_firstline /^\w\d{4}/ |
| format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ |
| time_format %m%d %H:%M:%S.%N |
| path /var/log/kube-apiserver.log |
| pos_file /var/log/gcp-kube-apiserver.log.pos |
| tag kube-apiserver |
| </source> |
| |
| # Example: |
| # I0204 06:55:31.872680 5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui |
| <source> |
| @type tail |
| format multiline |
| multiline_flush_interval 5s |
| format_firstline /^\w\d{4}/ |
| format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ |
| time_format %m%d %H:%M:%S.%N |
| path /var/log/kube-controller-manager.log |
| pos_file /var/log/gcp-kube-controller-manager.log.pos |
| tag kube-controller-manager |
| </source> |
| |
| # Example: |
| # W0204 06:49:18.239674 7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312] |
| <source> |
| @type tail |
| format multiline |
| multiline_flush_interval 5s |
| format_firstline /^\w\d{4}/ |
| format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ |
| time_format %m%d %H:%M:%S.%N |
| path /var/log/kube-scheduler.log |
| pos_file /var/log/gcp-kube-scheduler.log.pos |
| tag kube-scheduler |
| </source> |
| |
| # Example: |
| # I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from path /etc/gce.conf |
| <source> |
| @type tail |
| format multiline |
| multiline_flush_interval 5s |
| format_firstline /^\w\d{4}/ |
| format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ |
| time_format %m%d %H:%M:%S.%N |
| path /var/log/glbc.log |
| pos_file /var/log/gcp-glbc.log.pos |
| tag glbc |
| </source> |
| |
| # Example: |
| # I0603 15:31:05.793605 6 cluster_manager.go:230] Reading config from path /etc/gce.conf |
| <source> |
| @type tail |
| format multiline |
| multiline_flush_interval 5s |
| format_firstline /^\w\d{4}/ |
| format1 /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<message>.*)/ |
| time_format %m%d %H:%M:%S.%N |
| path /var/log/cluster-autoscaler.log |
| pos_file /var/log/gcp-cluster-autoscaler.log.pos |
| tag cluster-autoscaler |
| </source> |
| |
| # Logs from systemd-journal for interesting services. |
| # TODO(random-liu): Keep this for compatibility, remove this after |
| # cri container runtime rolls out. |
| <source> |
| @type systemd |
| filters [{ "_SYSTEMD_UNIT": "docker.service" }] |
| pos_file /var/log/gcp-journald-docker.pos |
| read_from_head true |
| tag docker |
| </source> |
| |
| <source> |
| @type systemd |
| filters [{ "_SYSTEMD_UNIT": "{{ container_runtime }}.service" }] |
| pos_file /var/log/gcp-journald-container-runtime.pos |
| read_from_head true |
| tag container-runtime |
| </source> |
| |
| <source> |
| @type systemd |
| filters [{ "_SYSTEMD_UNIT": "kubelet.service" }] |
| pos_file /var/log/gcp-journald-kubelet.pos |
| read_from_head true |
| tag kubelet |
| </source> |
| |
| <source> |
| @type systemd |
| filters [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }] |
| pos_file /var/log/gcp-journald-node-problem-detector.pos |
| read_from_head true |
| tag node-problem-detector |
| </source> |
| |
| # BEGIN_NODE_JOURNAL |
| # Whether to include node-journal or not is determined when starting the |
| # cluster. It is not changed when the cluster is already running. |
| <source> |
| @type systemd |
| pos_file /var/log/gcp-journald.pos |
| read_from_head true |
| tag node-journal |
| </source> |
| |
| <filter node-journal> |
| @type grep |
| <exclude> |
| key _SYSTEMD_UNIT |
| pattern ^(docker|{{ container_runtime }}|kubelet|node-problem-detector)\.service$ |
| </exclude> |
| </filter> |
| # END_NODE_JOURNAL |
| monitoring.conf: |- |
| # This source is used to acquire approximate process start timestamp, |
| # which purpose is explained before the corresponding output plugin. |
| <source> |
| @type exec |
| command /bin/sh -c 'date +%s' |
| tag process_start |
| time_format %Y-%m-%d %H:%M:%S |
| keys process_start_timestamp |
| </source> |
| |
| # This filter is used to convert process start timestamp to integer |
| # value for correct ingestion in the prometheus output plugin. |
| <filter process_start> |
| @type record_transformer |
| enable_ruby true |
| auto_typecast true |
| <record> |
| process_start_timestamp ${record["process_start_timestamp"].to_i} |
| </record> |
| </filter> |
| output.conf: |- |
| # This match is placed before the all-matching output to provide metric |
| # exporter with a process start timestamp for correct exporting of |
| # cumulative metrics to Stackdriver. |
| <match process_start> |
| @type prometheus |
| |
| <metric> |
| type gauge |
| name process_start_time_seconds |
| desc Timestamp of the process start in seconds |
| key process_start_timestamp |
| </metric> |
| </match> |
| |
| # This filter allows to count the number of log entries read by fluentd |
| # before they are processed by the output plugin. This in turn allows to |
| # monitor the number of log entries that were read but never sent, e.g. |
| # because of liveness probe removing buffer. |
| <filter **> |
| @type prometheus |
| <metric> |
| type counter |
| name logging_entry_count |
| desc Total number of log entries generated by either application containers or system components |
| </metric> |
| </filter> |
| |
| # This section is exclusive for k8s_container logs. Those come with |
| # 'stderr'/'stdout' tags. |
| # TODO(instrumentation): Reconsider this workaround later. |
| # Trim the entries which exceed slightly less than 100KB, to avoid |
| # dropping them. It is a necessity, because Stackdriver only supports |
| # entries that are up to 100KB in size. |
| <filter {stderr,stdout}> |
| @type record_transformer |
| enable_ruby true |
| <record> |
| message ${record['message'].length > 100000 ? "[Trimmed]#{record['message'][0..100000]}..." : record['message']} |
| </record> |
| </filter> |
| |
| # Do not collect fluentd's own logs to avoid infinite loops. |
| <match fluent.**> |
| @type null |
| </match> |
| |
| # Add a unique insertId to each log entry that doesn't already have it. |
| # This helps guarantee the order and prevent log duplication. |
| <filter **> |
| @type add_insert_ids |
| </filter> |
| |
| # This section is exclusive for k8s_container logs. These logs come with |
| # 'stderr'/'stdout' tags. |
| # We use a separate output stanza for 'k8s_node' logs with a smaller buffer |
| # because node logs are less important than user's container logs. |
| <match {stderr,stdout}> |
| @type google_cloud |
| |
| # Try to detect JSON formatted log entries. |
| detect_json true |
| # Collect metrics in Prometheus registry about plugin activity. |
| enable_monitoring true |
| monitoring_type prometheus |
| # Allow log entries from multiple containers to be sent in the same request. |
| split_logs_by_tag false |
| # Set the buffer type to file to improve the reliability and reduce the memory consumption |
| buffer_type file |
| buffer_path /var/log/fluentd-buffers/kubernetes.containers.buffer |
| # Set queue_full action to block because we want to pause gracefully |
| # in case of the off-the-limits load instead of throwing an exception |
| buffer_queue_full_action block |
| # Set the chunk limit conservatively to avoid exceeding the recommended |
| # chunk size of 5MB per write request. |
| buffer_chunk_limit 512k |
| # Cap the combined memory usage of this buffer and the one below to |
| # 512KiB/chunk * (6 + 2) chunks = 4 MiB |
| buffer_queue_limit 6 |
| # Never wait more than 5 seconds before flushing logs in the non-error case. |
| flush_interval 5s |
| # Never wait longer than 30 seconds between retries. |
| max_retry_wait 30 |
| # Disable the limit on the number of retries (retry forever). |
| disable_retry_limit |
| # Use multiple threads for processing. |
| num_threads 2 |
| use_grpc true |
| # Skip timestamp adjustment as this is in a controlled environment with |
| # known timestamp format. This helps with CPU usage. |
| adjust_invalid_timestamps false |
| </match> |
| |
| # Attach local_resource_id for 'k8s_node' monitored resource. |
| <filter **> |
| @type record_transformer |
| enable_ruby true |
| <record> |
| "logging.googleapis.com/local_resource_id" ${"k8s_node.#{ENV['NODE_NAME']}"} |
| </record> |
| </filter> |
| |
| # This section is exclusive for 'k8s_node' logs. These logs come with tags |
| # that are neither 'stderr' or 'stdout'. |
| # We use a separate output stanza for 'k8s_container' logs with a larger |
| # buffer because user's container logs are more important than node logs. |
| <match **> |
| @type google_cloud |
| |
| detect_json true |
| enable_monitoring true |
| monitoring_type prometheus |
| # Allow entries from multiple system logs to be sent in the same request. |
| split_logs_by_tag false |
| detect_subservice false |
| buffer_type file |
| buffer_path /var/log/fluentd-buffers/kubernetes.system.buffer |
| buffer_queue_full_action block |
| buffer_chunk_limit 512k |
| buffer_queue_limit 2 |
| flush_interval 5s |
| max_retry_wait 30 |
| disable_retry_limit |
| num_threads 2 |
| use_grpc true |
| # Skip timestamp adjustment as this is in a controlled environment with |
| # known timestamp format. This helps with CPU usage. |
| adjust_invalid_timestamps false |
| </match> |
| metadata: |
| name: fluentd-gcp-config-v1.2.5 |
| namespace: kube-system |
| labels: |
| addonmanager.kubernetes.io/mode: Reconcile |