Add documentation for help user deploy the Satellite as Gateway (#90)
diff --git a/configs/satellite_config.yaml b/configs/satellite_config.yaml
index 8957b57..4ac809b 100644
--- a/configs/satellite_config.yaml
+++ b/configs/satellite_config.yaml
@@ -127,7 +127,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -153,7 +153,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -179,7 +179,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -205,7 +205,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -247,7 +247,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -273,7 +273,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -299,7 +299,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -329,7 +329,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -359,7 +359,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -389,7 +389,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
@@ -419,7 +419,7 @@
# The maximum buffer event size.
event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
# The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:1}
+ partition: ${SATELLITE_QUEUE_PARTITION:4}
processor:
filters:
sender:
diff --git a/dist/bin/startup.cmd b/dist/bin/startup.cmd
new file mode 100644
index 0000000..9636836
--- /dev/null
+++ b/dist/bin/startup.cmd
@@ -0,0 +1,27 @@
+@REM
+@REM Licensed to the Apache Software Foundation (ASF) under one or more
+@REM contributor license agreements. See the NOTICE file distributed with
+@REM this work for additional information regarding copyright ownership.
+@REM The ASF licenses this file to You under the Apache License, Version 2.0
+@REM (the "License"); you may not use this file except in compliance with
+@REM the License. You may obtain a copy of the License at
+@REM
+@REM http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing, software
+@REM distributed under the License is distributed on an "AS IS" BASIS,
+@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@REM See the License for the specific language governing permissions and
+@REM limitations under the License.
+
+@echo off
+
+setlocal
+set SATELLITE_PROCESS_TITLE=Skywalking-Satellite
+set SATELLITE_HOME=%~dp0%..
+
+for /F %%i in ('where /r "%SATELLITE_HOME%\bin" skywalking-satellite*windows*') do ( set SATELLITE_EXEC=%%i)
+
+start "%SATELLITE_PROCESS_TITLE%" %SATELLITE_EXEC% start --config="%SATELLITE_HOME%\configs\satellite_config.yaml"
+
+endlocal
diff --git a/docs/en/FAQ/README.md b/docs/en/FAQ/README.md
deleted file mode 100644
index 20b7448..0000000
--- a/docs/en/FAQ/README.md
+++ /dev/null
@@ -1,2 +0,0 @@
-## FAQ
-- [What is the performance of the Satellite?](performance.md)
diff --git a/docs/en/FAQ/performance.md b/docs/en/FAQ/performance.md
deleted file mode 100644
index 0d8cd41..0000000
--- a/docs/en/FAQ/performance.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# What is the performance of the Satellite?
-## Performance
-The performance reduction of the mmap-queue is mainly due to the file persistent operation to ensure data stability. However, the queue is used to collect some core telemetry data. We will continue to optimize the performance of this queue.
-
-- 0.5 core supported 3000 ops throughput with memory queue.
-- 0.5 core supported 1500 ops throughput with the memory mapped queue(Ensure data stability).
-
-
-## Details
-### Testing environment
-1. machine:
- - cpu: INTEL Xeon E5-2650 V4 12C 2.2GHZ * 2
- - memory: INVENTEC PC4-19200 * 8
- - harddisk: INVENTEC SATA 4T 7.2K * 8
-2. Kafka:
- - region: the same region with the test machine in Baidu Cloud.
- - version.: 0.1.1.0
-3. The input plugin: grpc-nativelog-receiver
-4. resource limit:
- - cpu: 500m(0.5 core)
- - memory: 100M
-
-### Performance Test With Memory Queue
-| Qps |stack memory in use| heap memory in use |no-heap memory in use |
-| ---- | ---- | ---- | ---- |
-| 400 | 2.13M | 11M |83K|
-| 800 | 2.49M | 13.4M |83K|
-| 1200 | 2.72M | 13.4M |83K|
-| 1600 | 2.85M | 16.2M |83K|
-| 2000 | 2.92M | 17.6M |83K|
-| 2400 | 2.98M | 18.3M |83K|
-| 2800 | 3.54M | 26.8M |83K|
-| 3000 | 3.34M | 28M |83K|
-
-### Performance Test With Mmap Queue
-| Qps |stack memory in use| heap memory in use |no-heap memory in use |
-| ---- | ---- | ---- | ---- |
-| 400 | 2.39M | 9.5M |83K|
-| 800 | 2.43M | 12.1M |83K|
-| 1200 | 2.49M | 12M |83K|
-| 1600 | 2.62M | 13.3M |83K|
diff --git a/docs/en/setup/README.md b/docs/en/setup/README.md
index dd333bf..b74c027 100644
--- a/docs/en/setup/README.md
+++ b/docs/en/setup/README.md
@@ -19,10 +19,15 @@
## Examples
You can quickly build your satellite according to the following examples:
-1. [Forward Agent Data to OAP Server](examples/forward-agent-data-to-oap-server/README.md)
-2. [Transmit Log to Kafka](examples/transmit-log-to-kafka/README.md)
-3. [GRPC Load Balance Client](examples/grpc-load-balance-client/README.md)
-3. [High Throughput Queue](examples/high-throughput-queue/README.md)
+### Deploy
+
+1. [Deploy on Linux and Windows](examples/deploy/linux-windows/README.md)
+2. [Deploy on Kubernetes](examples/deploy/kubernetes/README.md)
+
+### More Use Cases
+
+1. [Transmit Log to Kafka](examples/feature/transmit-log-to-kafka/README.md)
+2. [Enable/Disable Channel](examples/feature/enable-disable-channel/README.md)
## satellite_config.yaml
The core concept behind this setting file is, SkyWalking Satellite is based on pure modularization design. End user can switch or assemble the collector features by their own requirements.
@@ -34,3 +39,7 @@
## Advanced feature document link list
1. [Overriding settings](./configuration/override-settings.md) in satellite_config.yaml is supported
+
+## Performance
+
+1. [ALS Load Balance](performance/als-load-balance/README.md).
diff --git a/docs/en/setup/examples/deploy/kubernetes/README.md b/docs/en/setup/examples/deploy/kubernetes/README.md
new file mode 100644
index 0000000..dcbc295
--- /dev/null
+++ b/docs/en/setup/examples/deploy/kubernetes/README.md
@@ -0,0 +1,28 @@
+# Deploy on Kubernetes
+
+It could help you run the Satellite as a gateway in Kubernetes environment.
+
+## Install
+
+We recommend install the Satellite by `helm`, follow command below, it could start the latest release version of SkyWalking Backend, UI and Satellite.
+
+```shell
+export SKYWALKING_RELEASE_NAME=skywalking # change the release name according to your scenario
+export SKYWALKING_RELEASE_NAMESPACE=default # change the namespace to where you want to install SkyWalking
+export REPO=skywalking
+
+helm repo add ${REPO} https://apache.jfrog.io/artifactory/skywalking-helm
+helm install "${SKYWALKING_RELEASE_NAME}" ${REPO}/skywalking -n "${SKYWALKING_RELEASE_NAMESPACE}" \
+ --set oap.image.tag=8.8.1 \
+ --set oap.storageType=elasticsearch \
+ --set ui.image.tag=8.8.1 \
+ --set elasticsearch.imageTag=6.8.6 \
+ --set satellite.enabled=true \
+ --set satellite.image.tag=v0.4.0
+```
+
+## Change Address
+
+After the Satellite and Backend started, need to change the address from agent/node. Then the satellite could load balance the request from agent/node to OAP backend.
+
+Such as in Java Agent, you should change the property value in `collector.backend_service` forward to this: `skywalking-satellite.${SKYWALKING_RELEASE_NAMESPACE}:11800`.
diff --git a/docs/en/setup/examples/grpc-load-balance-client/README.md b/docs/en/setup/examples/deploy/linux-windows/README.md
similarity index 74%
rename from docs/en/setup/examples/grpc-load-balance-client/README.md
rename to docs/en/setup/examples/deploy/linux-windows/README.md
index 3e2d573..691cf35 100644
--- a/docs/en/setup/examples/grpc-load-balance-client/README.md
+++ b/docs/en/setup/examples/deploy/linux-windows/README.md
@@ -1,14 +1,22 @@
-# GRPC Load Balance Client
+# Deploy on Linux and Windows
-GRPC client support connect to multiple server address, and use `round-robin` policy for load-balance server before send each request.
+It could help you run the Satellite as a gateway in Linux or Windows instance.
-## Server Discovery
+## Install
+
+### Download
+
+Download the latest release version from [SkyWalking Release Page](https://skywalking.apache.org/downloads/#SkyWalkingSatellite).
+
+### Change OAP Server addresses
+
+Update the OAP Server address in the config file, then satellite could connect to them and use `round-robin` policy for load-balance server before send each request.
Support two ways to locate the server list, using `finder_type` to change the type to find:
1. `static`: Define the server address list.
2. `kubernetes`: Define kubernetes pod/service/endpoint, it could be found addresses and dynamic update automatically.
-### Static server list
+#### Static server list
You could see there define two server address and split by ",".
@@ -41,7 +49,7 @@
tls_key_file: ${SATELLITE_GRPC_TLS_KEY_FILE:""}
```
-### Kubernetes selector
+#### Kubernetes selector
Using `kubernetes_config` to define the address's finder.
@@ -86,3 +94,15 @@
# The TLS key file path.
tls_key_file: ${SATELLITE_GRPC_TLS_KEY_FILE:""}
```
+
+### Start Satellite
+
+Execute the script `bin/startup.sh`(linux) or `bin/startup.cmd`(windows) to start. Then It could start these port:
+1. gRPC port(`11800`): listen the gRPC request, It could handle request from SkyWalking Agent protocol and Envoy ALS/Metrics protocol.
+2. Prometheus(`1234`): listen the HTTP request, It could get all `SO11Y` metrics from `/metrics` endpoint using Prometheus format.
+
+## Change Address
+
+After the satellite start, need to change the address from agent/node. Then the satellite could load balance the request from agent/node to OAP backend.
+
+Such as in Java Agent, you should change the property value in `collector.backend_service` forward to the satellite gRPC port.
diff --git a/docs/en/setup/examples/feature/enable-disable-channel/README.md b/docs/en/setup/examples/feature/enable-disable-channel/README.md
new file mode 100644
index 0000000..871d63a
--- /dev/null
+++ b/docs/en/setup/examples/feature/enable-disable-channel/README.md
@@ -0,0 +1,11 @@
+# Enable/Disable Channel
+
+Different channels mean that different protocols can be transparently transmitted to upstream services(OAP).
+
+## Config
+
+In the Satellite configuration, a channel is represented under the configured `pipes`. By default, we open all channels and process all known protocols.
+
+You could **delete** the channel if you don't want to receive and transmit in satellite.
+
+After restart the satellite service, then the channel what you delete is disable.
\ No newline at end of file
diff --git a/docs/en/setup/examples/transmit-log-to-kafka/README.md b/docs/en/setup/examples/feature/transmit-log-to-kafka/README.md
similarity index 100%
rename from docs/en/setup/examples/transmit-log-to-kafka/README.md
rename to docs/en/setup/examples/feature/transmit-log-to-kafka/README.md
diff --git a/docs/en/setup/examples/transmit-log-to-kafka/satellite_config.yaml b/docs/en/setup/examples/feature/transmit-log-to-kafka/satellite_config.yaml
similarity index 100%
rename from docs/en/setup/examples/transmit-log-to-kafka/satellite_config.yaml
rename to docs/en/setup/examples/feature/transmit-log-to-kafka/satellite_config.yaml
diff --git a/docs/en/setup/examples/forward-agent-data-to-oap-server/README.md b/docs/en/setup/examples/forward-agent-data-to-oap-server/README.md
deleted file mode 100644
index 5cce412..0000000
--- a/docs/en/setup/examples/forward-agent-data-to-oap-server/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Forward Agent Data to OAP Server
-
-Using Satellite to receive the SkyWalking protocols from agent, and transport data to the SkyWalking backend or another Satellite.
-
-## Protocols
-Support these protocols transport via `gRPC`:
-1. Tracing
-2. Log
-3. Management
-4. CDS
-5. Event
-6. JVM
-7. Profile
-
-## Config
-
-Here is [config file](satellite_config.yaml), set out as follows:
-
-- Declare gRPC [server](../../plugins/server_grpc-server.md) and [client](../../plugins/client_grpc-client.md) to receive and transmit data.
-- Declare each protocol gatherer and sender to transmit protocol via [pipeline](../../configuration/pipe-plugins.md).
-- Expose [Self-Observability telemetry data](../../configuration/common.md#self-telemetry) to [Prometheus](../../plugins/server_prometheus-server.md).
\ No newline at end of file
diff --git a/docs/en/setup/examples/forward-agent-data-to-oap-server/satellite_config.yaml b/docs/en/setup/examples/forward-agent-data-to-oap-server/satellite_config.yaml
deleted file mode 100644
index b61c265..0000000
--- a/docs/en/setup/examples/forward-agent-data-to-oap-server/satellite_config.yaml
+++ /dev/null
@@ -1,249 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# The logger configuration.
-logger:
- # The log format pattern configuration.
- log_pattern: ${SATELLITE_LOGGER_LOG_PATTERN:%time [%level][%field] - %msg}
- # The time format pattern configuration.
- time_pattern: ${SATELLITE_LOGGER_TIME_PATTERN:2006-01-02 15:04:05.000}
- # The lowest level of printing allowed.
- level: ${SATELLITE_LOGGER_LEVEL:info}
-
-# The Satellite self telemetry configuration.
-telemetry:
- # The space concept for the deployment, such as the namespace concept in the Kubernetes.
- cluster: ${SATELLITE_TELEMETRY_CLUSTER:satellite-cluster}
- # The group concept for the deployment, such as the service resource concept in the Kubernetes.
- service: ${SATELLITE_TELEMETRY_SERVICE:satellite-service}
- # The minimum running unit, such as the pod concept in the Kubernetes.
- instance: ${SATELLITE_TELEMETRY_SERVICE:satellite-instance}
-
-# The sharing plugins referenced by the specific plugins in the different pipes.
-sharing:
- clients:
- - plugin_name: "grpc-client"
- # The gRPC server address (default localhost:11800).
- server_addr: ${SATELLITE_GRPC_CLIENT:127.0.0.1:11800}
- # The TLS switch
- enable_TLS: ${SATELLITE_GRPC_ENABLE_TLS:false}
- # The file path of client.pem. The config only works when opening the TLS switch.
- client_pem_path: ${SATELLITE_GRPC_CLIENT_PEM_PATH:"client.pem"}
- # The file path of client.key. The config only works when opening the TLS switch.
- client_key_path: ${SATELLITE_GRPC_CLIENT_KEY_PATH:"client.key"}
- # InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name.
- insecure_skip_verify: ${SATELLITE_GRPC_INSECURE_SKIP_VERIFY:false}
- # The file path oca.pem. The config only works when opening the TLS switch.
- ca_pem_path: ${SATELLITE_grpc_CA_PEM_PATH:"ca.pem"}
- # How frequently to check the connection(second)
- check_period: ${SATELLITE_GRPC_CHECK_PERIOD:5}
- # The auth value when send request
- authentication: ${SATELLITE_GRPC_AUTHENTICATION:""}
- servers:
- - plugin_name: "grpc-server"
- # The address of grpc server.
- address: ${SATELLITE_GRPC_ADDRESS:":11800"}
- # The TLS cert file path.
- tls_cert_file: ${SATELLITE_GRPC_TLS_KEY_FILE:""}
- # The TLS key file path.
- tls_key_file: ${SATELLITE_GRPC_TLS_KEY_FILE:""}
- - plugin_name: "prometheus-server"
- # The prometheus server address.
- address: ${SATELLITE_PROMETHEUS_ADDRESS:":1234"}
- # The prometheus server metrics endpoint.
- endpoint: ${SATELLITE_PROMETHEUS_ENDPOINT:"/metrics"}
-
-# The working pipe configurations.
-pipes:
- - common_config:
- pipe_name: logpipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativelog-receiver"
- queue:
- plugin_name: "memory-queue"
- # The maximum buffer event size.
- event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- # The time interval between two flush operations. And the time unit is millisecond.
- flush_time: ${SATELLITE_LOGPIPE_SENDER_FLUSH_TIME:1000}
- # The maximum buffer elements.
- max_buffer_size: ${SATELLITE_LOGPIPE_SENDER_MAX_BUFFER_SIZE:200}
- # The minimum flush elements.
- min_flush_events: ${SATELLITE_LOGPIPE_SENDER_MIN_FLUSH_EVENTS:100}
- client_name: grpc-client
- forwarders:
- - plugin_name: nativelog-grpc-forwarder
- - common_config:
- pipe_name: managementpipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativemanagement-receiver"
- queue:
- plugin_name: "memory-queue"
- # The maximum buffer event size.
- event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- # The time interval between two flush operations. And the time unit is millisecond.
- flush_time: ${SATELLITE_LOGMANAGEMENT_SENDER_FLUSH_TIME:1000}
- # The maximum buffer elements.
- max_buffer_size: ${SATELLITE_LOGMANAGEMENT_SENDER_MAX_BUFFER_SIZE:20}
- # The minimum flush elements.
- min_flush_events: ${SATELLITE_LOGMANAGEMENT_SENDER_MIN_FLUSH_EVENTS:1}
- client_name: grpc-client
- forwarders:
- - plugin_name: nativemanagement-grpc-forwarder
- - common_config:
- pipe_name: tracingpipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativetracing-receiver"
- queue:
- plugin_name: "memory-queue"
- # The maximum buffer event size.
- event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- # The time interval between two flush operations. And the time unit is millisecond.
- flush_time: ${SATELLITE_TRACINGPIPE_SENDER_FLUSH_TIME:1000}
- # The maximum buffer elements.
- max_buffer_size: ${SATELLITE_TRACINGPIPE_SENDER_MAX_BUFFER_SIZE:200}
- # The minimum flush elements.
- min_flush_events: ${SATELLITE_TRACINGPIPE_SENDER_MIN_FLUSH_EVENTS:1}
- client_name: grpc-client
- forwarders:
- - plugin_name: nativetracing-grpc-forwarder
- - common_config:
- pipe_name: profilepipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativeprofile-receiver"
- queue:
- plugin_name: "memory-queue"
- # The maximum buffer event size.
- event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- # The time interval between two flush operations. And the time unit is millisecond.
- flush_time: ${SATELLITE_PROFILEPIPE_SENDER_FLUSH_TIME:1000}
- # The maximum buffer elements.
- max_buffer_size: ${SATELLITE_PROFILEPIPE_SENDER_MAX_BUFFER_SIZE:200}
- # The minimum flush elements.
- min_flush_events: ${SATELLITE_PROFILEPIPE_SENDER_MIN_FLUSH_EVENTS:1}
- client_name: grpc-client
- forwarders:
- - plugin_name: nativeprofile-grpc-forwarder
- - common_config:
- pipe_name: cdspipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativecds-receiver"
- queue:
- plugin_name: "none-queue"
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- client_name: grpc-client
- forwarders:
- - plugin_name: nativecds-grpc-forwarder
- - common_config:
- pipe_name: eventpipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativeevent-receiver"
- queue:
- plugin_name: "memory-queue"
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- # The time interval between two flush operations. And the time unit is millisecond.
- flush_time: ${SATELLITE_EVENTPIPE_SENDER_FLUSH_TIME:1000}
- # The maximum buffer elements.
- max_buffer_size: ${SATELLITE_EVENTPIPE_SENDER_MAX_BUFFER_SIZE:200}
- # The minimum flush elements.
- min_flush_events: ${SATELLITE_EVENTPIPE_SENDER_MIN_FLUSH_EVENTS:1}
- client_name: grpc-client
- forwarders:
- - plugin_name: nativeevent-grpc-forwarder
- - common_config:
- pipe_name: jvmpipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativejvm-receiver"
- queue:
- plugin_name: "memory-queue"
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- # The time interval between two flush operations. And the time unit is millisecond.
- flush_time: ${SATELLITE_JVMPIPE_SENDER_FLUSH_TIME:1000}
- # The maximum buffer elements.
- max_buffer_size: ${SATELLITE_JVMPIPE_SENDER_MAX_BUFFER_SIZE:200}
- # The minimum flush elements.
- min_flush_events: ${SATELLITE_JVMPIPE_SENDER_MIN_FLUSH_EVENTS:1}
- client_name: grpc-client
- forwarders:
- - plugin_name: nativejvm-grpc-forwarder
- - common_config:
- pipe_name: meterpipe
- gatherer:
- server_name: "grpc-server"
- receiver:
- plugin_name: "grpc-nativemeter-receiver"
- queue:
- plugin_name: "memory-queue"
- processor:
- filters:
- sender:
- fallbacker:
- plugin_name: none-fallbacker
- # The time interval between two flush operations. And the time unit is millisecond.
- flush_time: ${SATELLITE_METERPIPE_SENDER_FLUSH_TIME:1000}
- # The maximum buffer elements.
- max_buffer_size: ${SATELLITE_METERPIPE_SENDER_MAX_BUFFER_SIZE:200}
- # The minimum flush elements.
- min_flush_events: ${SATELLITE_METERPIPE_SENDER_MIN_FLUSH_EVENTS:1}
- client_name: grpc-client
- forwarders:
- - plugin_name: nativemeter-grpc-forwarder
diff --git a/docs/en/setup/examples/high-throughput-queue/README.md b/docs/en/setup/examples/high-throughput-queue/README.md
deleted file mode 100644
index a403b7f..0000000
--- a/docs/en/setup/examples/high-throughput-queue/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-# High Throughput Queue
-
-High throughput queues can send messages to multiple channels by using `round-robin` policy for load-balance, and consume messages from each queue in parallel.
-This prevents the single-threaded sender from performing network IO, blocking the `receiver`/`fetcher` to the queue.
-
-## Config
-
-Both existing two queues(`memory`, `mmap`) support this feature, add the `partition` to the queue config, default is `1`.
-
-```yaml
-queue:
- plugin_name: "memory-queue"
- # The maximum buffer event size.
- event_buffer_size: ${SATELLITE_QUEUE_EVENT_BUFFER_SIZE:5000}
- # The partition count of queue.
- partition: ${SATELLITE_QUEUE_PARTITION:2}
-```
-
-Following the config, we create a partitioned queue, have two sub-queue, each sub-queue has 5000 buffer.
diff --git a/docs/en/setup/performance/als-load-balance/README.md b/docs/en/setup/performance/als-load-balance/README.md
new file mode 100644
index 0000000..2acb3c4
--- /dev/null
+++ b/docs/en/setup/performance/als-load-balance/README.md
@@ -0,0 +1,45 @@
+# ALS Load Balance
+
+Using satellite as a load balancer in envoy and OAP can effectively prevent the problem of unbalanced messages received by OAP.
+
+In this case, we mainly use memory queues for intermediate data storage.
+
+Deference Envoy Count, OAP performance could impact the Satellite transmit performance.
+
+|Envoy Instance|Concurrent User|ALS OPS|Satellite CPU|Satellite Memory|
+|--------------|---------------|-------|-------------|----------------|
+|150|100|~50K|1.2C|0.5-1.0G|
+|150|300|~80K|1.8C|1.0-1.5G|
+|300|100|~50K|1.4C|0.8-1.2G|
+|300|300|~100K|2.2C|1.3-2.0G|
+|800|100|~50K|1.5C|0.9-1.5G|
+|800|300|~100K|2.6C|1.7-2.7G|
+|1500|100|~50K|1.7C|1.4-2.4G|
+|1500|300|~100K|2.7C|2.3-3.0G|
+|2300|150|~50K|1.8C|1.9-3.1G|
+|2300|300|~90K|2.5C|2.3-4.0G|
+|2300|500|~110K|3.2C|2.8-4.7G|
+
+## Detail
+
+### Environment
+
+Using GKE Environment, helm to build cluster.
+
+|Module|Version|Replicate Count|CPU Limit|Memory Limit|Description|
+|------|-------|---------------|---------|------------|-----------|
+|OAP|8.9.0|6|12C|32Gi|Using ElasticSearch as Storage|
+|Satellite|0.4.0|1|8C|16Gi||
+|ElasticSearch|7.5.1|3|8|16Gi||
+
+### Setting
+
+800 Envoy, 100K QPS ALS.
+
+|Module|Environment Config|Use Value|Default Value|Description|Recommend Value|
+|------|------------------|---------|-------------|-----------|--------------|
+|Satellite|SATELLITE_QUEUE_PARTITION|50|4|Support several goroutines concurrently to consume the queue|Satellite CPU number * 4-6, It could help improve throughput, but the default value also could handle `800` Envoy Instance and `100K` QPS ALS message. |
+|Satellite|SATELLITE_QUEUE_EVENT_BUFFER_SIZE|3000|1000|The size of the queue in each concurrency|This is related to the number of Envoys. If the number of Envoys is large, it is recommended to increase the value.|
+|Satellite|SATELLITE_ENVOY_ALS_V3_PIPE_RECEIVER_FLUSH_TIME|3000|1000|When the Satellite receives the message, how long(millisecond) will the ALS message be merged into an Event.|If a certain time delay is accepted, the value can be adjusted larger, which can effectively reduce CPU usage and make the Satellite more stable|
+|Satellite|SATELLITE_ENVOY_ALS_V3_PIPE_SENDER_FLUSH_TIME|3000|1000|How long(millisecond) is the memory queue data for each Goroutine to be summarized and sent to OAP|This depends on the amount of data in your queue, you can keep it consistent with `SATELLITE_ENVOY_ALS_V3_PIPE_RECEIVER_FLUSH_TIME`|
+|OAP|SW_CORE_GRPC_MAX_CONCURRENT_CALL|50|4|A link between Satellite and OAP, how many requests parallelism is supported|Same with `SATELLITE_QUEUE_PARTITION` in Satellite|
diff --git a/docs/menu.yml b/docs/menu.yml
index b0c783b..0ef394b 100644
--- a/docs/menu.yml
+++ b/docs/menu.yml
@@ -49,14 +49,22 @@
path: /en/setup/configuration/override-settings
- name: Examples
catalog:
- - name: Forward Agent Data to OAP Server
- path: /en/setup/examples/forward-agent-data-to-oap-server/readme
- - name: Transmit Log to Kafka
- path: /en/setup/examples/transmit-log-to-kafka/readme
- - name: GRPC Load Balance Client
- path: /en/setup/examples/grpc-load-balance-client/readme
- - name: High Throughput Queue
- path: /en/setup/examples/high-throughput-queue/readme
+ - name: Deploy
+ catalog:
+ - name: Deploy on Linux and Windows
+ path: /en/setup/examples/deploy/linux-windows/readme
+ - name: Deploy on Kubernetes
+ path: /en/setup/examples/deploy/kubernetes/readme
+ - name: More Use Cases
+ catalog:
+ - name: Transmit Log to Kafka
+ path: /en/setup/examples/feature/transmit-log-to-kafka/readme
+ - name: Enable/Disable Channel
+ path: /en/setup/examples/feature/enable-disable-channel/readme
+ - name: Performance
+ catalog:
+ - name: ALS Load Balance
+ path: /en/setup/performance/als-load-balance/readme
- name: Plugins
catalog:
- name: Client
@@ -163,5 +171,3 @@
catalog:
- name: How to compile SkyWalking Satellite
path: /en/guides/compile/How-to-compile
- - name: FAQs
- path: /en/FAQ/readme
diff --git a/internal/satellite/config/loader_test.go b/internal/satellite/config/loader_test.go
index e8fafb6..f9ba166 100644
--- a/internal/satellite/config/loader_test.go
+++ b/internal/satellite/config/loader_test.go
@@ -165,7 +165,7 @@
"commonfields_pipe_name": "logpipe",
"plugin_name": "memory-queue",
"event_buffer_size": 5000,
- "partition": 1,
+ "partition": 4,
},
},
Processor: &processor.ProcessorConfig{
diff --git a/plugins/queue/queues_windows.go b/plugins/queue/queues_windows.go
index 1bd507f..4ac056c 100644
--- a/plugins/queue/queues_windows.go
+++ b/plugins/queue/queues_windows.go
@@ -22,9 +22,11 @@
import (
"github.com/apache/skywalking-satellite/plugins/queue/api"
"github.com/apache/skywalking-satellite/plugins/queue/memory"
+ "github.com/apache/skywalking-satellite/plugins/queue/none"
)
var queues = []api.Queue{
// Please register the queue plugins available on Windows platform here.
new(memory.Queue),
+ new(none.Queue),
}
