Merge remote-tracking branch 'origin/dev'
diff --git a/docs-python/dev/scripts/gen_ref_pages/index.html b/docs-python/dev/scripts/gen_ref_pages/index.html
index d5f0f76..577eec7 100644
--- a/docs-python/dev/scripts/gen_ref_pages/index.html
+++ b/docs-python/dev/scripts/gen_ref_pages/index.html
@@ -3168,7 +3168,7 @@
init_mathjax();
</script>
<!-- End of mathjax configuration --><div class="jupyter-wrapper">
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=c1338937">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=160d20f1">
<div class="jp-Cell-inputWrapper">
<div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
</div>
@@ -3190,7 +3190,7 @@
</div>
</div>
</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=2b846d27">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=c7c8c995">
<div class="jp-Cell-inputWrapper">
<div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
</div>
@@ -3217,7 +3217,7 @@
</div>
</div>
</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=8a71a9e0">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=cf21051f">
<div class="jp-Cell-inputWrapper">
<div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
</div>
@@ -3244,7 +3244,7 @@
</div>
</div>
</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=7a22e556">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=45e096d6">
<div class="jp-Cell-inputWrapper">
<div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
</div>
@@ -3271,7 +3271,7 @@
</div>
</div>
</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs">
-<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=e7fa7c91">
+<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs" id="cell-id=f99690bf">
<div class="jp-Cell-inputWrapper">
<div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
</div>
@@ -3344,7 +3344,7 @@
</div>
</div>
</div>
-<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=29a3c064">
+<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell" id="cell-id=2b3f923a">
<div class="jp-Cell-inputWrapper">
<div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
</div>
diff --git a/docs-python/dev/sitemap.xml b/docs-python/dev/sitemap.xml
index f246800..07db782 100644
--- a/docs-python/dev/sitemap.xml
+++ b/docs-python/dev/sitemap.xml
@@ -2,242 +2,242 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/getting-started/developing/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/getting-started/first-steps/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/getting-started/quickstart/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/client/client/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/client/config/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/client/credential_provider/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/endpoint/endpoint/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/endpoint/exceptions/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/endpoint/api/data_lake_measure/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/endpoint/api/data_stream/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/endpoint/api/version/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/function_zoo/river_function/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/function_handler/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/registration/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/streampipes_function/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/broker/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/broker_handler/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/consumer/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/output_collector/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/publisher/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/kafka/kafka_consumer/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/kafka/kafka_message_fetcher/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/kafka/kafka_publisher/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/nats/nats_consumer/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/broker/nats/nats_publisher/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/utils/async_iter_handler/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/utils/data_stream_context/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/utils/data_stream_generator/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/functions/utils/function_context/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/common/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/container/data_lake_measures/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/container/data_streams/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/container/resource_container/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/container/versions/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/data_lake_measure/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/data_series/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/data_stream/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/exceptions/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/function_definition/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/query_result/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/resource/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/reference/model/resource/version/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/scripts/gen_ref_pages/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/tutorials/1-introduction-to-streampipes-python-client/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/tutorials/2-extracting-data-from-the-streampipes-data-lake/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/tutorials/3-getting-live-data-from-the-streampipes-data-stream/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>https://streampipes.apache.org/docs/docs/python/latest/latest/tutorials/4-using-online-machine-learning-on-a-streampipes-data-stream/</loc>
- <lastmod>2024-03-23</lastmod>
+ <lastmod>2024-03-30</lastmod>
<changefreq>daily</changefreq>
</url>
</urlset>
diff --git a/docs-python/dev/sitemap.xml.gz b/docs-python/dev/sitemap.xml.gz
index dc24031..1a3a953 100644
--- a/docs-python/dev/sitemap.xml.gz
+++ b/docs-python/dev/sitemap.xml.gz
Binary files differ
diff --git a/docs/pe/org.apache.streampipes.connect.iiot.adapters.oi4.md b/docs/pe/org.apache.streampipes.connect.iiot.adapters.oi4.md
index 6cb2a02..6758119 100644
--- a/docs/pe/org.apache.streampipes.connect.iiot.adapters.oi4.md
+++ b/docs/pe/org.apache.streampipes.connect.iiot.adapters.oi4.md
@@ -62,7 +62,7 @@
You should provide information about the sensor you want to connect to. This can be achieved in two ways:
-a) **By Type**: Specify the type of sensor you want to connect to, e.g., `'VVB001'`. <\br>
+a) **By Type**: Specify the type of sensor you want to connect to, e.g., `'VVB001'`. </br>
b) **By IODD**: Simply upload the IODD description of the respective sensor. Please note: This feature is not yet available! If you're interested in this feature, please notify us through the mailing list or GitHub discussions and share your use case with us.
### Selected Sensors
diff --git a/website-v2/blog/2024-03-27-anomaly-detection-with-python-functions.md b/website-v2/blog/2024-03-27-anomaly-detection-with-python-functions.md
new file mode 100644
index 0000000..963e31d
--- /dev/null
+++ b/website-v2/blog/2024-03-27-anomaly-detection-with-python-functions.md
@@ -0,0 +1,269 @@
+---
+title: Anomaly Detection with StreamPipes Functions in Python and ONNX
+author: Tim Bossenmaier
+authorURL: "https://github.com/bossenti"
+authorImageURL: "/img/bossenmaier.png"
+---
+
+
+Apache StreamPipes saves the day when it comes to connecting to data sources in the IIoT world. Want to do more with
+your IIoT data than just analyze it in a dashboard? If so, this blog post is for you! We'll show you how to extract
+historical data from StreamPipes, use it to train a machine learning model, bring the model back to StreamPipes using
+ONNX, and apply the model to live data.
+<center>
+<img class="blog-image" style={{maxWidth:'75%'}} src="/img/blog/2024-03-26/prediction-analysis.png" alt="anomaly-detection"/><br/>
+</center>
+
+<!--truncate-->
+
+## Motivation
+
+With this blogpost we want to illustrate how one can easily extract historical IIoT data collected with StreamPipes,
+train a machine learning model on this data and bringing the model back to StreamPipes with the interoperability
+standard [ONNX](https://onnx.ai) to make inference in live data.
+
+A very common use case in the area of IIoT is the detection of anomalies, so we want to tackle this challenge in this
+article as well. We will use data generated by
+the [Machine Data Simulator](https://streampipes.apache.org/docs/pe/org.apache.streampipes.connect.iiot.adapters.simulator.machine/)
+adapter. More specifically, we will focus on the `flowrate` data, which consists of various sensor values coming from a
+water pipe system. Our goal is to keep an eye on the parameter `volume_flow`, which represents the current volume flow
+in
+cubic meters/second. For this parameter, we want to detect anomalies that could indicate problems such as leaks,
+blockages, etc.
+
+To get the concerned data, we simply need to create an instance of the machine data simulator and persist the data in
+the data lake:
+
+
+
+## Set Up & Prepare Python Client
+
+As a prerequisite, we need to install the StreamPipes Python client and all other dependencies,
+
+```bash
+pip install git+https://github.com/apache/streampipes.git#subdirectory=streampipes-client-python
+pip install scikit-learn==1.4.0 skl2onnx==1.16.0 onnxruntime==1.17.1
+```
+
+The next step is to configure and initialize an instance of the client.
+
+```python
+import os
+from streampipes.client import StreamPipesClient
+from streampipes.client.config import StreamPipesClientConfig
+from streampipes.client.credential_provider import StreamPipesApiKeyCredentials
+
+os.environ["BROKER-HOST"] = "localhost"
+os.environ["KAFKA-PORT"] = "9094" # When using Kafka as message broker
+
+config = StreamPipesClientConfig(
+ credential_provider=StreamPipesApiKeyCredentials(
+ username="admin@streampipes.apache.org",
+ api_key="TOKEN",
+ ),
+ host_address="localhost",
+ https_disabled=True,
+ port=80
+)
+
+client = StreamPipesClient(client_config=config)
+```
+
+In case you have never worked with the Python client before and have problems to get started,
+please have a look at
+our [tutorial](https://streampipes.apache.org/docs/docs/python/latest/tutorials/1-introduction-to-streampipes-python-client/).
+
+If you already have an ONNX model and are only interested in applying it with StreamPipes on a data stream, you can skip
+the following section.
+
+## Model Training with Historic Data
+
+As said above, the aim of our model is to detect anomalies of the `volume_flow` parameter. For this task, we will use
+[Isolation Forests](https://en.wikipedia.org/wiki/Isolation_forest). Please note that the focus of the tutorial is not
+on training the model, so please be patient even though the training is very simplified and lacks important preparation
+steps such as standardization.
+
+As a first step, lets query the `flowrate` data from the StreamPipes data lake and extract the values of `volume_flow`
+as a feature:
+
+```python
+flowrate_df = client.dataLakeMeasureApi.get("flow-rate").to_pandas()
+X = flowrate_df["volume_flow"].values.reshape(-1, 1).astype("float32")
+```
+
+As a next step, we can already train our model with the historic data:
+
+```python
+from sklearn.ensemble import IsolationForest
+
+model = IsolationForest(contamination=0.01)
+model.fit(X)
+```
+
+The `contamination` parameter models the proportion of outliers in the data. See
+the [scikit-learn](https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.IsolationForest.html)
+documentation for more information.
+
+Here you can see how this simple model performs:
+
+<img src="/img/blog/2024-03-26/prediction-analysis.png"/>
+
+<p></p>
+
+This doesn't look too bad, right? Let's continue by converting our model to the ONNX representation.
+
+```python
+from onnxconverter_common import FloatTensorType
+from skl2onnx import to_onnx
+
+model_onnx = to_onnx(
+ model,
+ initial_types=[('input', FloatTensorType([None, X.shape[1]]))],
+ target_opset={'ai.onnx.ml': 3, 'ai.onnx': 15, '': 15}
+)
+
+with open("isolation_forest.onnx", "wb") as f:
+ f.write(model_onnx.SerializeToString())
+```
+
+## Model Inference with Live Data
+
+Utilizing a pre-trained model within StreamPipes becomes seamless with the ONNX interoperability standard, enabling
+effortless application of your existing model on live data streams.
+
+Interacting with live data from StreamPipes is facilitated through StreamPipes functions. Below, we'll create a Python
+StreamPipes function that leverages an ONNX model to generate predictions for each incoming event, making the results
+accessible as a data stream within StreamPipes for subsequent steps.
+
+So let's create an `ONNXFunction` that is capable of applying a model in ONNX representation to a StreamPipes data
+stream.
+If you'd like to read more details about how functions are defined, refer
+to [our tutorial](https://streampipes.apache.org/docs/docs/python/latest/tutorials/3-getting-live-data-from-the-streampipes-data-stream/).
+
+```python
+import numpy as np
+import onnxruntime as rt
+
+from streampipes.functions.broker.broker_handler import get_broker_description
+from streampipes.functions.streampipes_function import StreamPipesFunction
+from streampipes.functions.utils.data_stream_generator import create_data_stream, RuntimeType
+from streampipes.functions.utils.function_context import FunctionContext
+from streampipes.model.resource import FunctionDefinition, DataStream
+
+from typing import Dict, Any, List
+
+
+class ONNXFunction(StreamPipesFunction):
+
+ def __init__(self, feature_names: list[str], input_stream: DataStream):
+ output_stream = create_data_stream(
+ name="flowrate-prediction",
+ attributes={
+ "is_anomaly": RuntimeType.BOOLEAN.value
+ },
+ broker=get_broker_description(input_stream)
+ )
+
+ function_definition = FunctionDefinition(
+ consumed_streams=[input_stream.element_id]
+ ).add_output_data_stream(output_stream)
+
+ self.feature_names = feature_names
+ self.input_name = None
+ self.output_name = None
+ self.session = None
+
+ super().__init__(function_definition=function_definition)
+
+ ...
+```
+
+First, we need to take care about the data stream that is required to send the predictions from our function to
+StreamPipes. Thus, we create a dedicated output data stream which we need to provide with the attributes our event will
+consist of (a timestamp attribute is always added automatically). This output data stream needs to be registered at the
+function definition which is to be passed to the parent class. Lastly, we need to define some instance variables that
+are mainly required for the ONNX runtime.
+
+Next, we need to ensure that ONNX runtime session is created on start up. Thus, we need to invoke an InferenceSession
+and retrieving the corresponding configuration parameters:
+
+```python
+class ONNXFunction(StreamPipesFunction):
+
+ ...
+
+ def onServiceStarted(self, context: FunctionContext) -> None:
+ self.session = rt.InferenceSession(
+ path_or_bytes="isolation_forest.onnx",
+ providers=rt.get_available_providers(),
+ )
+ self.input_name = self.session.get_inputs()[0].name
+ self.output_name = self.session.get_outputs()[0].name
+
+ ...
+
+```
+
+Lastly, we need to implement the inference logic that is applied to every event.
+If you have brought up your own model, you need to adapt line `10-13`:
+
+```python jsx {10-13} showLineNumbers
+class ONNXFunction(StreamPipesFunction):
+
+ ...
+
+ def onEvent(self, event: Dict[str, Any], streamId: str) -> None:
+ feature_vector = []
+ for feature in self.feature_names:
+ feature_vector.append(event[feature])
+
+ prediction = self.session.run(
+ [self.output_name],
+ {self.input_name: np.expand_dims(np.array(feature_vector), axis=0).astype("float32")}
+ )[0]
+
+ output = {
+ "is_anomaly": int(prediction[0]) == -1
+ }
+
+ self.add_output(
+ stream_id=self.function_definition.get_output_stream_ids()[0],
+ event=output
+ )
+
+ def onServiceStopped(self) -> None:
+ pass
+```
+
+Having the function code in place, we can start the function with the following:
+
+```python
+from streampipes.functions.registration import Registration
+from streampipes.functions.function_handler import FunctionHandler
+
+stream = [
+ stream
+ for stream
+ in client.dataStreamApi.all()
+ if stream.name == "flow-rate"
+][0]
+
+function = ONNXFunction(
+ feature_names=["volume_flow"],
+ input_stream=stream
+)
+
+registration = Registration()
+registration.register(function)
+function_handler = FunctionHandler(registration, client)
+function_handler.initializeFunctions()
+```
+
+We can now access the live values of the prediction in the StreamPipes UI, e.g., in the pipeline editor.
+
+<img src="/img/blog/2024-03-26/tutorial-prediction-data-stream.png"/>
+
+<p></p>
+
+From here on you can further work with the prediction events in StreamPipes, e.g., by sending notifications
+to [MS Teams](https://streampipes.apache.org/docs/next/pe/org.apache.streampipes.sinks.notifications.jvm.msteams/).
diff --git a/website-v2/package-lock.json b/website-v2/package-lock.json
index 221eebb..21fdca1 100644
--- a/website-v2/package-lock.json
+++ b/website-v2/package-lock.json
@@ -4277,12 +4277,12 @@
}
},
"node_modules/body-parser": {
- "version": "1.20.1",
- "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz",
- "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==",
+ "version": "1.20.2",
+ "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz",
+ "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==",
"dependencies": {
"bytes": "3.1.2",
- "content-type": "~1.0.4",
+ "content-type": "~1.0.5",
"debug": "2.6.9",
"depd": "2.0.0",
"destroy": "1.2.0",
@@ -4290,7 +4290,7 @@
"iconv-lite": "0.4.24",
"on-finished": "2.4.1",
"qs": "6.11.0",
- "raw-body": "2.5.1",
+ "raw-body": "2.5.2",
"type-is": "~1.6.18",
"unpipe": "1.0.0"
},
@@ -4469,12 +4469,18 @@
}
},
"node_modules/call-bind": {
- "version": "1.0.2",
- "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
- "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
+ "version": "1.0.7",
+ "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz",
+ "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==",
"dependencies": {
- "function-bind": "^1.1.1",
- "get-intrinsic": "^1.0.2"
+ "es-define-property": "^1.0.0",
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.4",
+ "set-function-length": "^1.2.1"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -4950,9 +4956,9 @@
"integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A=="
},
"node_modules/cookie": {
- "version": "0.5.0",
- "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz",
- "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==",
+ "version": "0.6.0",
+ "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz",
+ "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==",
"engines": {
"node": ">= 0.6"
}
@@ -5510,6 +5516,22 @@
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz",
"integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ=="
},
+ "node_modules/define-data-property": {
+ "version": "1.1.4",
+ "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
+ "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
+ "dependencies": {
+ "es-define-property": "^1.0.0",
+ "es-errors": "^1.3.0",
+ "gopd": "^1.0.1"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/define-lazy-prop": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
@@ -5850,6 +5872,25 @@
"is-arrayish": "^0.2.1"
}
},
+ "node_modules/es-define-property": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
+ "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
+ "dependencies": {
+ "get-intrinsic": "^1.2.4"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
+ "node_modules/es-errors": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
+ "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/es-module-lexer": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.3.0.tgz",
@@ -6024,16 +6065,16 @@
}
},
"node_modules/express": {
- "version": "4.18.2",
- "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
- "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==",
+ "version": "4.19.2",
+ "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz",
+ "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==",
"dependencies": {
"accepts": "~1.3.8",
"array-flatten": "1.1.1",
- "body-parser": "1.20.1",
+ "body-parser": "1.20.2",
"content-disposition": "0.5.4",
"content-type": "~1.0.4",
- "cookie": "0.5.0",
+ "cookie": "0.6.0",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "2.0.0",
@@ -6338,9 +6379,9 @@
}
},
"node_modules/follow-redirects": {
- "version": "1.15.4",
- "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.4.tgz",
- "integrity": "sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==",
+ "version": "1.15.6",
+ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+ "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
"funding": [
{
"type": "individual",
@@ -6513,9 +6554,12 @@
}
},
"node_modules/function-bind": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
- "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+ "version": "1.1.2",
+ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
+ "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==",
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
},
"node_modules/gensync": {
"version": "1.0.0-beta.2",
@@ -6526,14 +6570,18 @@
}
},
"node_modules/get-intrinsic": {
- "version": "1.2.1",
- "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz",
- "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==",
+ "version": "1.2.4",
+ "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
+ "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
"dependencies": {
- "function-bind": "^1.1.1",
- "has": "^1.0.3",
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
"has-proto": "^1.0.1",
- "has-symbols": "^1.0.3"
+ "has-symbols": "^1.0.3",
+ "hasown": "^2.0.0"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -6679,6 +6727,17 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/gopd": {
+ "version": "1.0.1",
+ "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
+ "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
+ "dependencies": {
+ "get-intrinsic": "^1.1.3"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/ljharb"
+ }
+ },
"node_modules/got": {
"version": "9.6.0",
"resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz",
@@ -6778,11 +6837,11 @@
}
},
"node_modules/has-property-descriptors": {
- "version": "1.0.0",
- "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz",
- "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==",
+ "version": "1.0.2",
+ "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
+ "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
"dependencies": {
- "get-intrinsic": "^1.1.1"
+ "es-define-property": "^1.0.0"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
@@ -6818,6 +6877,17 @@
"node": ">=8"
}
},
+ "node_modules/hasown": {
+ "version": "2.0.2",
+ "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
+ "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+ "dependencies": {
+ "function-bind": "^1.1.2"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/hast-to-hyperscript": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/hast-to-hyperscript/-/hast-to-hyperscript-9.0.1.tgz",
@@ -8405,9 +8475,9 @@
}
},
"node_modules/object-inspect": {
- "version": "1.12.3",
- "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
- "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==",
+ "version": "1.13.1",
+ "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
+ "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
@@ -9604,9 +9674,9 @@
}
},
"node_modules/raw-body": {
- "version": "2.5.1",
- "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz",
- "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==",
+ "version": "2.5.2",
+ "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz",
+ "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==",
"dependencies": {
"bytes": "3.1.2",
"http-errors": "2.0.0",
@@ -10906,6 +10976,22 @@
"node": ">= 0.8.0"
}
},
+ "node_modules/set-function-length": {
+ "version": "1.2.2",
+ "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
+ "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==",
+ "dependencies": {
+ "define-data-property": "^1.1.4",
+ "es-errors": "^1.3.0",
+ "function-bind": "^1.1.2",
+ "get-intrinsic": "^1.2.4",
+ "gopd": "^1.0.1",
+ "has-property-descriptors": "^1.0.2"
+ },
+ "engines": {
+ "node": ">= 0.4"
+ }
+ },
"node_modules/setimmediate": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
@@ -10976,13 +11062,17 @@
}
},
"node_modules/side-channel": {
- "version": "1.0.4",
- "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz",
- "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==",
+ "version": "1.0.6",
+ "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
+ "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==",
"dependencies": {
- "call-bind": "^1.0.0",
- "get-intrinsic": "^1.0.2",
- "object-inspect": "^1.9.0"
+ "call-bind": "^1.0.7",
+ "es-errors": "^1.3.0",
+ "get-intrinsic": "^1.2.4",
+ "object-inspect": "^1.13.1"
+ },
+ "engines": {
+ "node": ">= 0.4"
},
"funding": {
"url": "https://github.com/sponsors/ljharb"
diff --git a/website-v2/static/img/blog/2024-03-26/prediction-analysis.png b/website-v2/static/img/blog/2024-03-26/prediction-analysis.png
new file mode 100644
index 0000000..e442bda
--- /dev/null
+++ b/website-v2/static/img/blog/2024-03-26/prediction-analysis.png
Binary files differ
diff --git a/website-v2/static/img/blog/2024-03-26/tutorial-prediction-data-stream.png b/website-v2/static/img/blog/2024-03-26/tutorial-prediction-data-stream.png
new file mode 100644
index 0000000..071a5f2
--- /dev/null
+++ b/website-v2/static/img/blog/2024-03-26/tutorial-prediction-data-stream.png
Binary files differ
