conf/functions_worker.yml - pulsar - Git at Google

 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
 # regarding copyright ownership.  The ASF licenses this file
 # to you under the Apache License, Version 2.0 (the
 # "License"); you may not use this file except in compliance
 # with the License.  You may obtain a copy of the License at
 #
 #   http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing,
 # software distributed under the License is distributed on an
 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
 #

 ####################
 # Worker Settings
 ####################

 workerId: standalone
 workerHostname: localhost
 workerPort: 6750
 workerPortTls: 6751

 # Configuration Store connection string
 configurationStoreServers: localhost:2181
 # ZooKeeper session timeout in milliseconds
 zooKeeperSessionTimeoutMillis: 30000
 # ZooKeeper operation timeout in seconds
 zooKeeperOperationTimeoutSeconds: 30

 ################################
 # Function package management
 ################################

 numFunctionPackageReplicas: 1
 downloadDirectory: download/pulsar_functions
 # Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
 # jvmGCMetricsLoggerClassName:

 #################################################################
 # Function metadata managment (assignment, scheduling, and etc)
 #################################################################

 # Configure the pulsar client used by function metadata management
 #
 # points
 # Whether to enable TLS when clients connect to broker
 useTls: false
 # For TLS:
 # brokerServiceUrl=pulsar+ssl://localhost:6651/
 pulsarServiceUrl: pulsar://localhost:6650
 # For TLS:
 # webServiceUrl=https://localhost:8443/
 pulsarWebServiceUrl: http://localhost:8080

 ############################################
 # security settings for pulsar broker client
 ############################################
 # The path to trusted certificates used by the Pulsar client to authenticate with Pulsar brokers
 # brokerClientTrustCertsFilePath:
 # Whether to enable the broker client authentication used by function workers to talk to brokers
 # brokerClientAuthenticationEnabled: false
 # the authentication plugin to be used by the pulsar client used in worker service
 # brokerClientAuthenticationPlugin:
 # the authentication parameter to be used by the pulsar client used in worker service
 # brokerClientAuthenticationParameters:

 # Bookie Authentication
 #
 # Authentication plugin to use when connecting to bookies
 # bookkeeperClientAuthenticationPlugin:
 # BookKeeper auth plugin implementatation specifics parameters name and values
 # bookkeeperClientAuthenticationParametersName:
 # Parameters for bookkeeper auth plugin
 # bookkeeperClientAuthenticationParameters:


 # pulsar topics used for function metadata management

 pulsarFunctionsNamespace: public/functions
 pulsarFunctionsCluster: standalone
 functionMetadataTopicName: metadata
 clusterCoordinationTopicName: coordinate
 # Should the metadata topic be compacted? Defaulting it to false for backwards compatibility
 useCompactedMetadataTopic: false
 # Number of threads to use for HTTP requests processing. Default is set to 8
 numHttpServerThreads: 8

 # function assignment and scheduler

 schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler"
 functionAssignmentTopicName: "assignments"
 failureCheckFreqMs: 30000
 rescheduleTimeoutMs: 60000
 # frequency at which to check if cluster needs rebalancing (set to -1 to disable)
 rebalanceCheckFreqSec: -1
 initialBrokerReconnectMaxRetries: 60
 assignmentWriteMaxRetries: 60
 instanceLivenessCheckFreqMs: 30000
 # Frequency how often worker performs compaction on function-topics
 topicCompactionFrequencySec: 1800
 # should the function context have pulsar admin client exposed? Default is disabled.
 exposeAdminClientEnabled: false


 ###############################
 # Function Runtime Management
 ###############################

 #### Process Runtime  ####
 # Pulsar function instances are launched as processes

 functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory
 functionRuntimeFactoryConfigs:
     # location of log files for functions
     logDirectory: logs/
     # change the jar location only when you put the java instance jar in a different location
     javaInstanceJarLocation:
     # change the python instance location only when you put the python instance jar in a different location
     pythonInstanceLocation:
     # change the extra dependencies location:
     extraFunctionDependenciesDir:

 #### Thread Runtime ####
 # Pulsar function instances are run as threads

 #functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory
 #functionRuntimeFactoryConfigs:
 #  # thread group name
 #  threadGroupName: "Thread Function Container Group"
 ## Set the pulsar client memory limit
 #pulsarClientMemoryLimit
 #    # the max memory in bytes the pulsar client can use
 #    absoluteValue:
 #    # the max memory the pulsar client can use as a percentage of max direct memory set for JVM
 #    percentOfMaxDirectMemory:

 #### Kubernetes Runtime ####
 # Pulsar function are deployed to Kubernetes

 #functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory
 #functionRuntimeFactoryConfigs:
 #    # uri to kubernetes cluster, leave it to empty and it will use the kubernetes settings in function worker
 #    k8Uri:
 #    # the kubernetes namespace to run the function instances. it is `default`, if this setting is left to be empty
 #    jobNamespace:
 #    # The Kubernetes pod name to run the function instances. It is set to
 #    # `pf-<tenant>-<namespace>-<function_name>-<random_uuid(8)>` if this setting is left to be empty
 #    jobName:
 #    # the docker image to run function instance. by default it is `apachepulsar/pulsar`
 #    pulsarDockerImageName:
 #    # the docker image to run function instance according to different configurations provided by users.
 #    # By default it is `apachepulsar/pulsar`.
 #    # e.g:
 #    # functionDockerImages:
 #    #   JAVA: JAVA_IMAGE_NAME
 #    #   PYTHON: PYTHON_IMAGE_NAME
 #    #   GO: GO_IMAGE_NAME
 #    functionDockerImages:
 #    # "The image pull policy for image used to run function instance. By default it is `IfNotPresent`
 #    imagePullPolicy: IfNotPresent
 #    # the root directory of pulsar home directory in `pulsarDockerImageName`. by default it is `/pulsar`.
 #    # if you are using your own built image in `pulsarDockerImageName`, you need to set this setting accordingly
 #    pulsarRootDir:
 #    # The config admin CLI allows users to customize the configuration of the admin cli tool, such as:
 #    # `/bin/pulsar-admin and /bin/pulsarctl`. By default it is `/bin/pulsar-admin`. If you want to use `pulsarctl`
 #    # you need to set this setting accordingly
 #    configAdminCLI:
 #    # this setting only takes effects if `k8Uri` is set to null. if your function worker is running as a k8 pod,
 #    # setting this to true is let function worker to submit functions to the same k8s cluster as function worker
 #    # is running. setting this to false if your function worker is not running as a k8 pod.
 #    submittingInsidePod: false
 #    # setting the pulsar service url that pulsar function should use to connect to pulsar
 #    # if it is not set, it will use the pulsar service url configured in worker service
 #    pulsarServiceUrl:
 #    # setting the pulsar admin url that pulsar function should use to connect to pulsar
 #    # if it is not set, it will use the pulsar admin url configured in worker service
 #    pulsarAdminUrl:
 #    # The flag indicates to install user code dependencies. (applied to python package)
 #    installUserCodeDependencies:
 #    # The repository that pulsar functions use to download python dependencies
 #    pythonDependencyRepository:
 #    # The repository that pulsar functions use to download extra python dependencies
 #    pythonExtraDependencyRepository:
 #    # the custom labels that function worker uses to select the nodes for pods
 #    customLabels:
 #    # The expected metrics collection interval, in seconds
 #    expectedMetricsCollectionInterval: 30
 #    # Kubernetes Runtime will periodically checkback on
 #    # this configMap if defined and if there are any changes
 #    # to the kubernetes specific stuff, we apply those changes
 #    changeConfigMap:
 #    # The namespace for storing change config map
 #    changeConfigMapNamespace:
 #    # The ratio cpu request and cpu limit to be set for a function/source/sink.
 #    # The formula for cpu request is cpuRequest = userRequestCpu / cpuOverCommitRatio
 #    cpuOverCommitRatio: 1.0
 #    # The ratio memory request and memory limit to be set for a function/source/sink.
 #    # The formula for memory request is memoryRequest = userRequestMemory / memoryOverCommitRatio
 #    memoryOverCommitRatio: 1.0
 #    # The port inside the function pod which is used by the worker to communicate with the pod
 #    grpcPort: 9093
 #    # The port inside the function pod on which prometheus metrics are exposed
 #    metricsPort: 9094
 #    # The directory inside the function pod where nar packages will be extracted
 #    narExtractionDirectory:
 #    # The classpath where function instance files stored
 #    functionInstanceClassPath:
 #    # the directory for dropping extra function dependencies
 #    # if it is not an absolute path, it is relative to `pulsarRootDir`
 #    extraFunctionDependenciesDir:
 #    # Additional memory padding added on top of the memory requested by the function per on a per instance basis
 #    percentMemoryPadding: 10

 ## A set of the minimum amount of resources functions must request.
 ## Support for this depends on function runtime.
 ## Only kubernetes runtime currently supports this.
 # functionInstanceMinResources:
 #   cpu: 1
 #   ram: 1073741824
 #   disk: 10737418240

 ## A set of the maximum amount of resources functions may request.
 ## Support for this depends on function runtime.
 ## Only kubernetes runtime currently supports this.
 # functionInstanceMaxResources:
 #   cpu: 16
 #   ram: 17179869184
 #   disk: 107374182400

 ## The full class-name of an instance of RuntimeCustomizer.
 ## This class receives the customRuntimeOptions string and can customize details of how the runtime operates.
 #runtimeCustomizerClassName: "org.apache.pulsar.functions.runtime.kubernetes.KubernetesManifestCustomizer"

 ## This config will pass to RuntimeCustomizer's initialize function to do initializing.
 #runtimeCustomizerConfig:
 #   extractLabels:
 #       extraLabel: value
 #   extraAnnotations:
 #       extraAnnotation: value
 #   nodeSelectorLabels:
 #       customLabel: value
 #   jobNamespace: namespace
 #   tolerations:
 #   - key: custom-key
 #     value: value
 #     effect: NoSchedule

 ## Config admin CLI
 #configAdminCLI:

 ############################################
 # security settings for worker service
 ############################################

 # Enforce authentication
 authenticationEnabled: false
 # Enforce authorization on accessing functions api
 authorizationEnabled: false
 # Set of authentication provider name list, which is a list of class names
 authenticationProviders:
 # Authorization provider fully qualified class-name
 authorizationProvider: org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider
 # Set of role names that are treated as "super-user", meaning they will be able to access any admin-api
 superUserRoles:

 #### tls configuration for worker service
 # Enable TLS
 tlsEnabled: false
 # Path for the TLS certificate file
 tlsCertificateFilePath:
 # Path for the TLS private key file
 tlsKeyFilePath:
 # Path for the trusted TLS certificate file
 tlsTrustCertsFilePath:
 # Accept untrusted TLS certificate from client
 tlsAllowInsecureConnection: false
 # Whether server hostname must match the common name of the certificate
 tlsEnableHostnameVerification: false
 # Tls cert refresh duration in seconds (set 0 to check on every new connection)
 tlsCertRefreshCheckDurationSec: 300

 ########################
 # State Management
 ########################

 # the service url points to bookkeeper table service
 # stateStorageServiceUrl: bk://localhost:4181

 ########################
 # Connectors
 ########################

 connectorsDirectory: ./connectors
 functionsDirectory: ./functions

 # Should connector config be validated during during submission
 validateConnectorConfig: false

 # Whether to initialize distributed log metadata by runtime.
 # If it is set to true, you must ensure that it has been initialized by "bin/pulsarinitialize-cluster-metadata" command.
 initializedDlogMetadata: false
	#
	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	#

	####################
	# Worker Settings
	####################

	workerId: standalone
	workerHostname: localhost
	workerPort: 6750
	workerPortTls: 6751

	# Configuration Store connection string
	configurationStoreServers: localhost:2181
	# ZooKeeper session timeout in milliseconds
	zooKeeperSessionTimeoutMillis: 30000
	# ZooKeeper operation timeout in seconds
	zooKeeperOperationTimeoutSeconds: 30

	################################
	# Function package management
	################################

	numFunctionPackageReplicas: 1
	downloadDirectory: download/pulsar_functions
	# Classname of Pluggable JVM GC metrics logger that can log GC specific metrics
	# jvmGCMetricsLoggerClassName:

	#################################################################
	# Function metadata managment (assignment, scheduling, and etc)
	#################################################################

	# Configure the pulsar client used by function metadata management
	#
	# points
	# Whether to enable TLS when clients connect to broker
	useTls: false
	# For TLS:
	# brokerServiceUrl=pulsar+ssl://localhost:6651/
	pulsarServiceUrl: pulsar://localhost:6650
	# For TLS:
	# webServiceUrl=https://localhost:8443/
	pulsarWebServiceUrl: http://localhost:8080

	############################################
	# security settings for pulsar broker client
	############################################
	# The path to trusted certificates used by the Pulsar client to authenticate with Pulsar brokers
	# brokerClientTrustCertsFilePath:
	# Whether to enable the broker client authentication used by function workers to talk to brokers
	# brokerClientAuthenticationEnabled: false
	# the authentication plugin to be used by the pulsar client used in worker service
	# brokerClientAuthenticationPlugin:
	# the authentication parameter to be used by the pulsar client used in worker service
	# brokerClientAuthenticationParameters:

	# Bookie Authentication
	#
	# Authentication plugin to use when connecting to bookies
	# bookkeeperClientAuthenticationPlugin:
	# BookKeeper auth plugin implementatation specifics parameters name and values
	# bookkeeperClientAuthenticationParametersName:
	# Parameters for bookkeeper auth plugin
	# bookkeeperClientAuthenticationParameters:


	# pulsar topics used for function metadata management

	pulsarFunctionsNamespace: public/functions
	pulsarFunctionsCluster: standalone
	functionMetadataTopicName: metadata
	clusterCoordinationTopicName: coordinate
	# Should the metadata topic be compacted? Defaulting it to false for backwards compatibility
	useCompactedMetadataTopic: false
	# Number of threads to use for HTTP requests processing. Default is set to 8
	numHttpServerThreads: 8

	# function assignment and scheduler

	schedulerClassName: "org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler"
	functionAssignmentTopicName: "assignments"
	failureCheckFreqMs: 30000
	rescheduleTimeoutMs: 60000
	# frequency at which to check if cluster needs rebalancing (set to -1 to disable)
	rebalanceCheckFreqSec: -1
	initialBrokerReconnectMaxRetries: 60
	assignmentWriteMaxRetries: 60
	instanceLivenessCheckFreqMs: 30000
	# Frequency how often worker performs compaction on function-topics
	topicCompactionFrequencySec: 1800
	# should the function context have pulsar admin client exposed? Default is disabled.
	exposeAdminClientEnabled: false


	###############################
	# Function Runtime Management
	###############################

	#### Process Runtime ####
	# Pulsar function instances are launched as processes

	functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.process.ProcessRuntimeFactory
	functionRuntimeFactoryConfigs:
	# location of log files for functions
	logDirectory: logs/
	# change the jar location only when you put the java instance jar in a different location
	javaInstanceJarLocation:
	# change the python instance location only when you put the python instance jar in a different location
	pythonInstanceLocation:
	# change the extra dependencies location:
	extraFunctionDependenciesDir:

	#### Thread Runtime ####
	# Pulsar function instances are run as threads

	#functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory
	#functionRuntimeFactoryConfigs:
	# # thread group name
	# threadGroupName: "Thread Function Container Group"
	## Set the pulsar client memory limit
	#pulsarClientMemoryLimit
	# # the max memory in bytes the pulsar client can use
	# absoluteValue:
	# # the max memory the pulsar client can use as a percentage of max direct memory set for JVM
	# percentOfMaxDirectMemory:

	#### Kubernetes Runtime ####
	# Pulsar function are deployed to Kubernetes

	#functionRuntimeFactoryClassName: org.apache.pulsar.functions.runtime.kubernetes.KubernetesRuntimeFactory
	#functionRuntimeFactoryConfigs:
	# # uri to kubernetes cluster, leave it to empty and it will use the kubernetes settings in function worker
	# k8Uri:
	# # the kubernetes namespace to run the function instances. it is `default`, if this setting is left to be empty
	# jobNamespace:
	# # The Kubernetes pod name to run the function instances. It is set to
	# # `pf-<tenant>-<namespace>-<function_name>-<random_uuid(8)>` if this setting is left to be empty
	# jobName:
	# # the docker image to run function instance. by default it is `apachepulsar/pulsar`
	# pulsarDockerImageName:
	# # the docker image to run function instance according to different configurations provided by users.
	# # By default it is `apachepulsar/pulsar`.
	# # e.g:
	# # functionDockerImages:
	# # JAVA: JAVA_IMAGE_NAME
	# # PYTHON: PYTHON_IMAGE_NAME
	# # GO: GO_IMAGE_NAME
	# functionDockerImages:
	# # "The image pull policy for image used to run function instance. By default it is `IfNotPresent`
	# imagePullPolicy: IfNotPresent
	# # the root directory of pulsar home directory in `pulsarDockerImageName`. by default it is `/pulsar`.
	# # if you are using your own built image in `pulsarDockerImageName`, you need to set this setting accordingly
	# pulsarRootDir:
	# # The config admin CLI allows users to customize the configuration of the admin cli tool, such as:
	# # `/bin/pulsar-admin and /bin/pulsarctl`. By default it is `/bin/pulsar-admin`. If you want to use `pulsarctl`
	# # you need to set this setting accordingly
	# configAdminCLI:
	# # this setting only takes effects if `k8Uri` is set to null. if your function worker is running as a k8 pod,
	# # setting this to true is let function worker to submit functions to the same k8s cluster as function worker
	# # is running. setting this to false if your function worker is not running as a k8 pod.
	# submittingInsidePod: false
	# # setting the pulsar service url that pulsar function should use to connect to pulsar
	# # if it is not set, it will use the pulsar service url configured in worker service
	# pulsarServiceUrl:
	# # setting the pulsar admin url that pulsar function should use to connect to pulsar
	# # if it is not set, it will use the pulsar admin url configured in worker service
	# pulsarAdminUrl:
	# # The flag indicates to install user code dependencies. (applied to python package)
	# installUserCodeDependencies:
	# # The repository that pulsar functions use to download python dependencies
	# pythonDependencyRepository:
	# # The repository that pulsar functions use to download extra python dependencies
	# pythonExtraDependencyRepository:
	# # the custom labels that function worker uses to select the nodes for pods
	# customLabels:
	# # The expected metrics collection interval, in seconds
	# expectedMetricsCollectionInterval: 30
	# # Kubernetes Runtime will periodically checkback on
	# # this configMap if defined and if there are any changes
	# # to the kubernetes specific stuff, we apply those changes
	# changeConfigMap:
	# # The namespace for storing change config map
	# changeConfigMapNamespace:
	# # The ratio cpu request and cpu limit to be set for a function/source/sink.
	# # The formula for cpu request is cpuRequest = userRequestCpu / cpuOverCommitRatio
	# cpuOverCommitRatio: 1.0
	# # The ratio memory request and memory limit to be set for a function/source/sink.
	# # The formula for memory request is memoryRequest = userRequestMemory / memoryOverCommitRatio
	# memoryOverCommitRatio: 1.0
	# # The port inside the function pod which is used by the worker to communicate with the pod
	# grpcPort: 9093
	# # The port inside the function pod on which prometheus metrics are exposed
	# metricsPort: 9094
	# # The directory inside the function pod where nar packages will be extracted
	# narExtractionDirectory:
	# # The classpath where function instance files stored
	# functionInstanceClassPath:
	# # the directory for dropping extra function dependencies
	# # if it is not an absolute path, it is relative to `pulsarRootDir`
	# extraFunctionDependenciesDir:
	# # Additional memory padding added on top of the memory requested by the function per on a per instance basis
	# percentMemoryPadding: 10

	## A set of the minimum amount of resources functions must request.
	## Support for this depends on function runtime.
	## Only kubernetes runtime currently supports this.
	# functionInstanceMinResources:
	# cpu: 1
	# ram: 1073741824
	# disk: 10737418240

	## A set of the maximum amount of resources functions may request.
	## Support for this depends on function runtime.
	## Only kubernetes runtime currently supports this.
	# functionInstanceMaxResources:
	# cpu: 16
	# ram: 17179869184
	# disk: 107374182400

	## The full class-name of an instance of RuntimeCustomizer.
	## This class receives the customRuntimeOptions string and can customize details of how the runtime operates.
	#runtimeCustomizerClassName: "org.apache.pulsar.functions.runtime.kubernetes.KubernetesManifestCustomizer"

	## This config will pass to RuntimeCustomizer's initialize function to do initializing.
	#runtimeCustomizerConfig:
	# extractLabels:
	# extraLabel: value
	# extraAnnotations:
	# extraAnnotation: value
	# nodeSelectorLabels:
	# customLabel: value
	# jobNamespace: namespace
	# tolerations:
	# - key: custom-key
	# value: value
	# effect: NoSchedule

	## Config admin CLI
	#configAdminCLI:

	############################################
	# security settings for worker service
	############################################

	# Enforce authentication
	authenticationEnabled: false
	# Enforce authorization on accessing functions api
	authorizationEnabled: false
	# Set of authentication provider name list, which is a list of class names
	authenticationProviders:
	# Authorization provider fully qualified class-name
	authorizationProvider: org.apache.pulsar.broker.authorization.PulsarAuthorizationProvider
	# Set of role names that are treated as "super-user", meaning they will be able to access any admin-api
	superUserRoles:

	#### tls configuration for worker service
	# Enable TLS
	tlsEnabled: false
	# Path for the TLS certificate file
	tlsCertificateFilePath:
	# Path for the TLS private key file
	tlsKeyFilePath:
	# Path for the trusted TLS certificate file
	tlsTrustCertsFilePath:
	# Accept untrusted TLS certificate from client
	tlsAllowInsecureConnection: false
	# Whether server hostname must match the common name of the certificate
	tlsEnableHostnameVerification: false
	# Tls cert refresh duration in seconds (set 0 to check on every new connection)
	tlsCertRefreshCheckDurationSec: 300

	########################
	# State Management
	########################

	# the service url points to bookkeeper table service
	# stateStorageServiceUrl: bk://localhost:4181

	########################
	# Connectors
	########################

	connectorsDirectory: ./connectors
	functionsDirectory: ./functions

	# Should connector config be validated during during submission
	validateConnectorConfig: false

	# Whether to initialize distributed log metadata by runtime.
	# If it is set to true, you must ensure that it has been initialized by "bin/pulsarinitialize-cluster-metadata" command.
	initializedDlogMetadata: false