[RELEASE] [skip-ci]merging 'release-0.60.0' into 'master'
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..4ac7a5b
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,339 @@
+variables:
+ REGISTRY_HOST: ipe-wim-gitlab.fzi.de:5000
+ IMAGE_NAME: $REGISTRY_HOST/$CI_PROJECT_PATH
+ DOCKER_HUB_IMAGE_NAME: streampipes/
+ MAVEN_CLI_OPTS: -DskipTests --batch-mode --errors --fail-at-end --show-version -DinstallAtEnd=true -DdeployAtEnd=true
+ HARBOR_REGISTRY_HOST: laus.fzi.de:8201
+ HARBOR_IMAGE_NAME: $HARBOR_REGISTRY_HOST/$CI_PROJECT_PATH
+ GIT_REPO_ORIGIN: ssh://git@ipe-wim-gitlab.fzi.de:2222
+ GIT_STRATEGY: clone
+
+
+stages:
+ - build
+ - github
+# - docker
+# - deploy
+ - release
+ - docker-hub
+
+build:
+ image: maven:3-jdk-8
+ stage: build
+ script:
+ - echo "$GPG_PRIVATE_KEY" | gpg --batch --import --passphrase "$GPG_PASSPHRASE"
+ - echo "$MAVEN_CREDENTIALS" > /root/.m2/settings.xml
+ - mvn clean package javadoc:aggregate -U -DskipTests
+ - export MVN_VERSION=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version | grep -v '\[')
+ - "echo $MVN_VERSION >> ./target/mvn_version"
+ artifacts:
+ paths:
+ - ./*/target/*.jar
+ - ./target/mvn_version
+ expire_in: 1 hour
+ except:
+ - /release-.*$/
+
+
+github:
+ image: maven:3-jdk-8
+ stage: github
+ script:
+ - git config --global user.email 'zehnder@fzi.de'
+ - git config --global user.name 'zehnder'
+ - git checkout $CI_COMMIT_REF_NAME
+ - git pull
+ - git remote add github https://$GITHUB_TOKEN:x-oauth-basic@github.com/streampipes/streampipes-pipeline-elements.git
+ - git push github $CI_COMMIT_REF_NAME
+ only:
+ - dev
+ - master
+
+
+#.docker_script: &docker_script
+# image: docker:17.06.0-ce
+# stage: docker
+# dependencies:
+# - build
+# script:
+# - export MVN_VERSION=$(cat ./target/mvn_version)
+# - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $REGISTRY_HOST
+# - docker login -u riemer -p $HARBOR_PASSWORD laus.fzi.de:8201
+# - docker build --no-cache --pull -t $IMAGE_NAME/$CONTAINER_NAME:latest -t $IMAGE_NAME/$CONTAINER_NAME:$MVN_VERSION -t $HARBOR_IMAGE_NAME/$CONTAINER_NAME:latest -t $HARBOR_IMAGE_NAME/$CONTAINER_NAME:$MVN_VERSION ./streampipes-$CONTAINER_NAME
+# - docker push $IMAGE_NAME/$CONTAINER_NAME:$MVN_VERSION
+# - docker push $IMAGE_NAME/$CONTAINER_NAME:latest
+# - docker push $HARBOR_IMAGE_NAME/$CONTAINER_NAME:$MVN_VERSION
+# - docker push $HARBOR_IMAGE_NAME/$CONTAINER_NAME:latest
+# only:
+# - dev
+
+.docker_hub: &docker_hub_script
+ image: docker:17.06.0-ce
+ stage: docker-hub
+ when: manual
+ script:
+ - export MVN_VERSION=$(cat ./target/mvn_version)
+ - docker login -u streampipesbuild -p $DOCKER_HUB_USER_PWD
+ - docker build --no-cache --pull -t $DOCKER_HUB_IMAGE_NAME/$CONTAINER_NAME:latest -t $DOCKER_HUB_IMAGE_NAME/$CONTAINER_NAME:$MVN_VERSION ./streampipes-$CONTAINER_NAME
+ - docker push $DOCKER_HUB_IMAGE_NAME/$CONTAINER_NAME:$MVN_VERSION
+ - docker push $DOCKER_HUB_IMAGE_NAME/$CONTAINER_NAME:latest
+ when: manual
+ only:
+ - master
+
+#deploy:
+# image: maven:3-jdk-8
+# stage: deploy
+# script:
+# - echo "$GPG_PRIVATE_KEY" | gpg --batch --import --passphrase "$GPG_PASSPHRASE"
+# - echo "$MAVEN_CREDENTIALS" > /root/.m2/settings.xml
+# - mvn deploy -DskipTests
+# artifacts:
+# paths:
+# - ./**/*.jar
+# only:
+# - dev
+
+start release:
+ image: maven:3-jdk-8
+ stage: release
+ script:
+ - echo "$MAVEN_CREDENTIALS" > /root/.m2/settings.xml
+ - git remote set-url origin $GIT_REPO_ORIGIN/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME.git
+ - eval $(ssh-agent -s)
+ - ssh-add <(echo "$SSH")
+ - mkdir ~/.ssh
+ - touch ~/.ssh/known_hosts
+ - ssh-keyscan -p 2222 -t rsa ipe-wim-gitlab.fzi.de > ~/.ssh/known_hosts
+ - git config --global user.email 'zehnder@fzi.de'
+ - git config --global user.name 'zehnder'
+ - git fetch
+ - git checkout master
+ - git checkout dev
+ - mvn -B jgitflow:release-start $MAVEN_CLI_OPTS
+ - git push origin --all
+ only:
+ - dev
+ when: manual
+
+finish release:
+ image: maven:3-jdk-8
+ stage: release
+ script:
+ - export MVN_VERSION=$(cat ./target/mvn_version)
+ - echo "$GPG_PRIVATE_KEY" | gpg --batch --import --passphrase "$GPG_PASSPHRASE"
+ - echo "$MAVEN_CREDENTIALS" > /root/.m2/settings.xml
+ - git remote set-url origin $GIT_REPO_ORIGIN/$CI_PROJECT_NAMESPACE/$CI_PROJECT_NAME.git
+ - git remote add github https://$GITHUB_TOKEN:x-oauth-basic@github.com/streampipes/streampipes-pipeline-elements.git
+ - eval $(ssh-agent -s)
+ - ssh-add <(echo "$SSH")
+ - mkdir ~/.ssh
+ - touch ~/.ssh/known_hosts
+ - ssh-keyscan -p 2222 -t rsa ipe-wim-gitlab.fzi.de > ~/.ssh/known_hosts
+ - git config --global user.email 'zehnder@fzi.de'
+ - git config --global user.name 'zehnder'
+ - git checkout master
+ - git checkout dev
+ - git checkout $CI_BUILD_REF_NAME
+ - mvn -B jgitflow:release-finish $MAVEN_CLI_OPTS
+ - git push origin --all
+ - git push origin --tags
+ - git checkout master
+ - git push github master
+ - git push github $MVN_VERSION
+ - git checkout dev
+ - git push github dev
+ when: manual
+ except:
+ - master
+
+#docker-processors-aggregation-flink:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-aggregation-flink"
+#
+#docker-processors-enricher-flink:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-enricher-flink"
+#
+#docker-processors-filters-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-filters-jvm"
+#
+#docker-processors-geo-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-geo-jvm"
+#
+#docker-processors-image-processing-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-image-processing-jvm"
+#
+#docker-processors-pattern-detection-flink:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-pattern-detection-flink"
+#
+#docker-processors-filters-siddhi:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-filters-siddhi"
+#
+#docker-processors-statistics-flink:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-statistics-flink"
+#
+#docker-processors-geo-flink:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-geo-flink"
+#
+#docker-processors-transformation-flink:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-transformation-flink"
+#
+#docker-processors-transformation-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "processors-transformation-jvm"
+#
+#docker-sinks-brokers-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sinks-brokers-jvm"
+#
+#docker-sinks-databases-flink:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sinks-databases-flink"
+#
+#docker-sinks-databases-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sinks-databases-jvm"
+#
+#docker-sinks-notifications-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sinks-notifications-jvm"
+#
+#docker-sinks-internal-jvm:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sinks-internal-jvm"
+#
+#docker-sources-vehicle-simulator:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sources-vehicle-simulator"
+#
+#docker-sources-random:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sources-random-data-generator"
+#
+#docker-sources-watertank-simulator:
+# <<: *docker_script
+# variables:
+# CONTAINER_NAME: "sources-watertank-simulator"
+
+
+docker-hub-processors-aggregation-flink:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-aggregation-flink"
+
+docker-hub-processors-enricher-flink:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-enricher-flink"
+
+docker-hub-processors-filters-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-filters-jvm"
+
+docker-hub-processors-geo-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-geo-jvm"
+
+docker-hub-processors-image-processing-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-image-processing-jvm"
+
+docker-hub-processors-pattern-detection-flink:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-pattern-detection-flink"
+
+docker-hub-processors-filters-siddhi:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-filters-siddhi"
+
+docker-hub-processors-statistics-flink:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-statistics-flink"
+
+docker-hub-processors-geo-flink:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-geo-flink"
+
+docker-hub-processors-transformation-flink:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-transformation-flink"
+
+docker-hub-processors-transformation-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "processors-transformation-jvm"
+
+docker-hub-sinks-brokers-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sinks-brokers-jvm"
+
+docker-hub-sinks-databases-flink:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sinks-databases-flink"
+
+docker-hub-sinks-databases-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sinks-databases-jvm"
+
+docker-hub-sinks-notifications-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sinks-notifications-jvm"
+
+docker-hub-sinks-internal-jvm:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sinks-internal-jvm"
+
+docker-hub-sources-vehicle-simulator:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sources-vehicle-simulator"
+
+docker-hub-sources-random:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sources-random-data-generator"
+
+docker-hub-sources-watertank-simulator:
+ <<: *docker_hub_script
+ variables:
+ CONTAINER_NAME: "sources-watertank-simulator"
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 483ebbb..0000000
--- a/Dockerfile
+++ /dev/null
@@ -1,8 +0,0 @@
-FROM anapsix/alpine-java
-
-EXPOSE 8090
-ENV CONSUL_LOCATION consul
-
-ADD ./target/streampipes-examples-jvm.jar /streampipes-examples-jvm.jar
-
-ENTRYPOINT ["java", "-jar", "/streampipes-examples-jvm.jar"]
diff --git a/README.md b/README.md
index 637b603..1283992 100644
--- a/README.md
+++ b/README.md
@@ -9,41 +9,64 @@
Read the full documentation at [https://docs.streampipes.org](https://docs.streampipes.org)
-### StreamPipes examples for standalone pipeline elements running directly on the JVM
+### StreamPipes Pipeline Elements
-This project includes examples for StreamPipes data processors and data sinks that do not use a specific runtime such
- as Apache Flink but run directly on the JVM in a single-host manner. These components are suitable for processing
- event streams with rather low frequency (e.g., up to a few thousand events per second)
+This project provides a library of several pipeline elements that can be used within the StreamPipes toolbox.
-Currently, the following example pipeline elements are available:
+Currently, the following pipeline elements are available:
+
+**Data Sources**
+* Watertank Simulator: A data simulator that replays data from an Industrial IoT use case. Several streams are provided such as flow rate, water level and more.
+* Vehicle Simulator: Provides a simulated stream that replays location-based real-time data of a vehicle.
+* Random Data Generator: Several streams that produce randomly generated data in an endless stream.
**Data Processors**
+* Aggregation
+* Count Aggregation
+* Event Rate
+* Timestamp Enricher
* Numerical Filter: Filters sensor values based on a configurable threshold value.
* Text Filter: Filters text-based fields by a given string value.
* Projection: Outputs a configurable subset of the fields available in an input event stream.
+* Geocoding
+* Google Routing
+* Increase
+* Peak Detection
+* Statistics Summary
+* Statistics Summary Window-Based
+* Field Converter
+* Field Hasher
+* Field Mapper
+* Measurement Unit Converter
+* Field Renamer
+
**Data Sinks**
* CouchDB: Stores events in an Apache CouchDB database.
* Dashboard: Can be used to display pipeline results in the real-time dashboard of the StreamPipes UI.
* Kafka Publisher: Publishes events to an Apache Kafka broker.
* Notification: Can be used to generate notifications that are shown in the notification center of the StreamPipes UI.
+* JMS
+* RabbitMQ
+* Elasticsearch
+* Dashboard
+* Notification
+* Email Notification
+* Slack Notification
+* OneSignal Notification
+
+Contact us if you are missing some pipeline elements!
### Getting started
-Currently, the StreamPipes core is available as a preview in form of ready-to-use Docker images.
+All modules contain a docker-compose template in the `deployment` folder of each module. Copy this into your StreamPipes docker-compose file (see the docs for more info) and start StreamPipes! All pipeline elements are automatically registered (but do not forget to install them under `Pipeline Element Installation`).
It's easy to get started:
-* Download the `docker-compose.yml` file from [https://www.github.com/streampipes/preview-docker](https://www.github.com/streampipes/preview-docker)
-* Follow the installation guide at [https://docs.streampipes.org/quick_start/installation](https://docs.streampipes.org/quick_start/installation)
-* Check the [tour](https://docs.streampipes.org/user_guide/features) and build your first pipeline!
-
-### Extending StreamPipes
-
-You can easily add your own data streams, processors or sinks.
-
-Check our developer guide at [https://docs.streampipes.org/developer_guide/introduction](https://docs.streampipes.org/developer_guide/introduction)
+* Download the installer script from [https://github.com/streampipes/streampipes-installer](https://github.com/streampipes/streampipes-installer)
+* Follow the installation guide at [https://docs.streampipes.org/docs/user-guide-installation](https://docs.streampipes.org/docs/user-guide-installation)
+* Check the [tour](https://docs.streampipes.org/docs/user-guide-tour) and build your first pipeline!
### Feedback
-We'd love to hear your feedback! Contact us at [mail.streampipes@gmail.com](mailto:mail.streampipes@gmail.com)
+We'd love to hear your feedback! Contact us at [feedback@streampipes.org](mailto:feedback@streampipes.org)
diff --git a/pom.xml b/pom.xml
index 5ce364e..15866d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
<groupId>org.streampipes</groupId>
<artifactId>streampipes-pipeline-elements</artifactId>
<packaging>pom</packaging>
- <version>0.55.3-SNAPSHOT</version>
+ <version>0.60.0</version>
<modules>
<module>streampipes-sinks-databases-jvm</module>
<module>streampipes-sinks-internal-jvm</module>
@@ -17,50 +17,114 @@
<module>streampipes-processors-enricher-flink</module>
<module>streampipes-sources-watertank-simulator</module>
<module>streampipes-sources-vehicle-simulator</module>
+ <module>streampipes-processors-transformation-flink</module>
+ <module>streampipes-processors-geo-jvm</module>
+ <module>streampipes-processors-statistics-flink</module>
+ <module>streampipes-processors-filters-siddhi</module>
+ <module>streampipes-processors-text-mining-flink</module>
+ <module>streampipes-sources-random-data-generator</module>
+ <module>streampipes-sinks-notifications-jvm</module>
+ <module>streampipes-pipeline-elements-shared</module>
+ <module>streampipes-processors-geo-flink</module>
+ <module>streampipes-processors-image-processing-jvm</module>
+ <module>streampipes-processors-transformation-jvm</module>
</modules>
<properties>
- <streampipes.version>0.55.3-SNAPSHOT</streampipes.version>
+ <streampipes.version>0.60.0</streampipes.version>
<lightcouch.version>0.1.8</lightcouch.version>
</properties>
- <dependencies>
- <dependency>
- <groupId>org.streampipes</groupId>
- <artifactId>streampipes-wrapper-standalone</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.streampipes</groupId>
- <artifactId>streampipes-sdk</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.streampipes</groupId>
- <artifactId>streampipes-container-standalone</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.streampipes</groupId>
- <artifactId>streampipes-config</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.streampipes</groupId>
- <artifactId>streampipes-dataformat-json</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.streampipes</groupId>
- <artifactId>streampipes-messaging-kafka</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.lightcouch</groupId>
- <artifactId>lightcouch</artifactId>
- <version>${lightcouch.version}</version>
- </dependency>
- </dependencies>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-commons</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sources</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-standalone</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-flink</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-dataformat-json</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-messaging-kafka</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-messaging-jms</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.lightcouch</groupId>
+ <artifactId>lightcouch</artifactId>
+ <version>${lightcouch.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-measurement-units</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-pipeline-elements-shared</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-test-utils</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-siddhi</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.5</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<build>
<pluginManagement>
@@ -78,11 +142,6 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
- <version>2.10.4</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
<version>3.0.0</version>
</plugin>
</plugins>
@@ -97,47 +156,12 @@
<goals>
<goal>jar</goal>
</goals>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- </configuration>
</execution>
</executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.3</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>org.streampipes.examples.jvm.ExamplesJvmInit</mainClass>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- </transformers>
- <filters>
-
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
- <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
- </exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
+ <configuration>
+ <failOnError>false</failOnError>
+ <additionalparam>-Xdoclint:none</additionalparam>
+ </configuration>
</plugin>
<plugin>
<groupId>external.atlassian.jgitflow</groupId>
@@ -150,9 +174,8 @@
<featureBranchPrefix>feature-</featureBranchPrefix>
<releaseBranchPrefix>release-</releaseBranchPrefix>
<hotfixBranchPrefix>hotfix-</hotfixBranchPrefix>
- <versionTagPrefix>version-</versionTagPrefix>
</flowInitContext>
- <noDeploy>false</noDeploy>
+ <noDeploy>true</noDeploy>
<autoVersionSubmodules>true</autoVersionSubmodules>
<pushReleases>false</pushReleases>
<localOnly>true</localOnly>
@@ -182,15 +205,13 @@
</configuration>
</plugin>
</plugins>
- <finalName>streampipes-examples-jvm</finalName>
-
</build>
<scm>
- <developerConnection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-jvm.git
+ <developerConnection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/streampipes-pipeline-elements.git
</developerConnection>
- <connection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-jvm.git</connection>
- <url>https://github.com/streampipes/pe-examples-jvm</url>
+ <connection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/streampipes-pipeline-elements.git</connection>
+ <url>https://github.com/streampipes/streampipes-pipeline-elements</url>
</scm>
<licenses>
diff --git a/streampipes-pipeline-elements-shared/pom.xml b/streampipes-pipeline-elements-shared/pom.xml
new file mode 100644
index 0000000..21faf99
--- /dev/null
+++ b/streampipes-pipeline-elements-shared/pom.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2018 FZI Forschungszentrum Informatik
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-pipeline-elements-shared</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
diff --git a/streampipes-pipeline-elements-shared/src/main/java/org/streampipes/pe/shared/PlaceholderExtractor.java b/streampipes-pipeline-elements-shared/src/main/java/org/streampipes/pe/shared/PlaceholderExtractor.java
new file mode 100644
index 0000000..7ea8522
--- /dev/null
+++ b/streampipes-pipeline-elements-shared/src/main/java/org/streampipes/pe/shared/PlaceholderExtractor.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.pe.shared;
+
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class PlaceholderExtractor {
+
+ private static final Pattern pattern = Pattern.compile("#[^#]*#");
+
+ public static String replacePlaceholders(String content, String json) {
+ List<String> placeholders = getPlaceholders(content);
+ JsonParser parser = new JsonParser();
+ JsonObject jsonObject = parser.parse(json).getAsJsonObject();
+
+ for(String placeholder : placeholders) {
+ String replacedValue = getPropertyValue(jsonObject, placeholder);
+ content = content.replaceAll(placeholder, replacedValue);
+ }
+
+ return content;
+ }
+
+ public static String replacePlaceholders(String content, Map<String, Object> event) {
+ List<String> placeholders = getPlaceholders(content);
+
+ for(String placeholder : placeholders) {
+ String replacedValue = getPropertyValue(event, placeholder);
+ content = content.replaceAll(placeholder, replacedValue);
+ }
+
+ return content;
+ }
+
+ private static String getPropertyValue(Map<String, Object> event, String placeholder) {
+ String key = placeholder.replaceAll("#", "");
+ return String.valueOf(event.get(key));
+ }
+
+ private static String getPropertyValue(JsonObject jsonObject, String placeholder) {
+ String jsonKey = placeholder.replaceAll("#", "");
+ return String.valueOf(jsonObject.get(jsonKey).getAsString());
+ }
+
+ private static List<String> getPlaceholders(String content) {
+ List<String> results = new ArrayList<>();
+ Matcher matcher = pattern.matcher(content);
+ while (matcher.find()) {
+ results.add(matcher.group());
+ }
+ return results;
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/Dockerfile b/streampipes-processors-aggregation-flink/Dockerfile
index d17f3af..77ed1ca 100644
--- a/streampipes-processors-aggregation-flink/Dockerfile
+++ b/streampipes-processors-aggregation-flink/Dockerfile
@@ -3,6 +3,6 @@
EXPOSE 8090
ENV CONSUL_LOCATION consul
-ADD ./target/streampipes-examples-sources.jar /streampipes-examples-sources.jar
+ADD target/streampipes-processors-aggregation-flink.jar /streampipes-processing-element-container.jar
-ENTRYPOINT ["java", "-jar", "/streampipes-examples-sources.jar"]
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-aggregation-flink/deployment/docker-compose.yml b/streampipes-processors-aggregation-flink/deployment/docker-compose.yml
new file mode 100644
index 0000000..d661a6c
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-aggregation-flink:
+ image: ${SP_PE_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-aggregation-flink:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-aggregation-flink/development/.env b/streampipes-processors-aggregation-flink/development/.env
new file mode 100644
index 0000000..dcc81bc
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/development/.env
@@ -0,0 +1,8 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6005
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_ELASTICSEARCH_HOST=localhost
+SP_FLINK_DEBUG=true
diff --git a/streampipes-processors-aggregation-flink/pom.xml b/streampipes-processors-aggregation-flink/pom.xml
index 8c5b9e9..a076765 100644
--- a/streampipes-processors-aggregation-flink/pom.xml
+++ b/streampipes-processors-aggregation-flink/pom.xml
@@ -1,18 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>streampipes-pipeline-elements</artifactId>
<groupId>org.streampipes</groupId>
- <version>0.55.3-SNAPSHOT</version>
+ <version>0.60.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>streampipes-processors-aggregation-flink</artifactId>
<properties>
- <streampipes.version>0.55.3-SNAPSHOT</streampipes.version>
<elasticsearch.version>5.2.2</elasticsearch.version>
</properties>
@@ -118,51 +115,27 @@
<artifactId>flink-cep-scala_2.11</artifactId>
<version>1.4.0</version>
</dependency>
+ <dependency>
+ <groupId>io.flinkspector</groupId>
+ <artifactId>flinkspector-datastream_2.11</artifactId>
+ <version>0.8.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
</dependencies>
<build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.0</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.10.4</version>
- </plugin>
- </plugins>
- </pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
@@ -200,86 +173,9 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>external.atlassian.jgitflow</groupId>
- <artifactId>jgitflow-maven-plugin</artifactId>
- <version>1.0-m5.1</version>
- <configuration>
- <flowInitContext>
- <masterBranchName>master</masterBranchName>
- <developBranchName>dev</developBranchName>
- <featureBranchPrefix>feature-</featureBranchPrefix>
- <releaseBranchPrefix>release-</releaseBranchPrefix>
- <hotfixBranchPrefix>hotfix-</hotfixBranchPrefix>
- <versionTagPrefix>version-</versionTagPrefix>
- </flowInitContext>
- <noDeploy>false</noDeploy>
- <autoVersionSubmodules>true</autoVersionSubmodules>
- <pushReleases>false</pushReleases>
- <localOnly>true</localOnly>
- <squash>false</squash>
- <scmCommentPrefix>[RELEASE] [skip-ci]</scmCommentPrefix>
- <enableSshAgent>true</enableSshAgent>
- </configuration>
- </plugin>
</plugins>
<finalName>streampipes-processors-aggregation-flink</finalName>
</build>
- <scm>
- <developerConnection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-flink.git
- </developerConnection>
- <connection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-flink.git</connection>
- <url>https://github.com/streampipes/pe-examples-flink</url>
- </scm>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
-
- <developers>
- <developer>
- <name>Dominik Riemer</name>
- <email>riemer@fzi.de</email>
- </developer>
- <developer>
- <name>Philipp Zehnder</name>
- <email>zehnder@fzi.de</email>
- </developer>
- </developers>
-
- <repositories>
- <repository>
- <id>laus</id>
- <name>nexus repository</name>
- <url>https://laus.fzi.de/nexus/content/repositories/public/</url>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </snapshots>
- </repository>
- </repositories>
-
- <distributionManagement>
- <repository>
- <id>sonatype</id>
- <name>Releases</name>
- <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
- </repository>
- <snapshotRepository>
- <id>deployment</id>
- <name>Internal Releases</name>
- <url>https://laus.fzi.de/nexus/content/repositories/snapshots/</url>
- </snapshotRepository>
- </distributionManagement>
-
</project>
\ No newline at end of file
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/AbstractAggregationProgram.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/AbstractAggregationProgram.java
new file mode 100644
index 0000000..29ee865
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/AbstractAggregationProgram.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.aggregation.flink;
+
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import org.streampipes.processors.aggregation.flink.config.AggregationFlinkConfig;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public abstract class AbstractAggregationProgram<B extends EventProcessorBindingParams> extends FlinkDataProcessorRuntime<B> {
+
+ public AbstractAggregationProgram(B params, boolean debug) {
+ super(params, debug);
+ }
+
+ public AbstractAggregationProgram(B params) {
+ super(params, false);
+ }
+
+ @Override
+ protected FlinkDeploymentConfig getDeploymentConfig() {
+ return new FlinkDeploymentConfig(AggregationFlinkConfig.JAR_FILE,
+ AggregationFlinkConfig.INSTANCE.getFlinkHost(), AggregationFlinkConfig.INSTANCE.getFlinkPort());
+ }
+
+}
+
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/AggregationFlinkInit.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/AggregationFlinkInit.java
index babd237..5438408 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/AggregationFlinkInit.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/AggregationFlinkInit.java
@@ -16,16 +16,17 @@
package org.streampipes.processors.aggregation.flink;
-import org.streampipes.processors.aggregation.flink.config.AggregationFlinkConfig;
-import org.streampipes.processors.aggregation.flink.processor.aggregation.AggregationController;
import org.streampipes.container.init.DeclarersSingleton;
import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.processors.aggregation.flink.config.AggregationFlinkConfig;
+import org.streampipes.processors.aggregation.flink.processor.aggregation.AggregationController;
public class AggregationFlinkInit extends StandaloneModelSubmitter {
public static void main(String[] args) {
DeclarersSingleton.getInstance()
.add(new AggregationController());
+ //.add(new CountController());
new AggregationFlinkInit().init(AggregationFlinkConfig.INSTANCE);
}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
index eb28d56..130704e 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/config/AggregationFlinkConfig.java
@@ -23,7 +23,7 @@
INSTANCE;
private SpConfig config;
- public static final String JAR_FILE = "./streampipes-examples-flink.jar";
+ public static final String JAR_FILE = "./streampipes-processing-element-container.jar";
private final static String service_id = "pe/org.streampipes.processors.aggregation.flink";
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/config/service b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/config/service
new file mode 100644
index 0000000..35fc574
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/config/service
@@ -0,0 +1,17 @@
+processors-aggregation-flink:
+ image: streampipes/processors-aggregation-flink:${SP_VERSION}
+ depends_on:
+ - "consul"
+# environment:
+# - SP_HOST=${SP_HOST}
+# - SP_PORT=${SP_PORT}
+# - SP_FLINK_HOST=${SP_FLINK_HOST}
+# - SP_FLINK_PORT=${SP_FLINK_PORT}
+# - SP_ELASTIC_HOST=${SP_ELASTIC_HOST}
+# - SP_ELASTIC_PORT=${SP_ELASTIC_PORT}
+# - SP_ELASTIC_PORT_REST=${SP_ELASTIC_PORT_REST}
+# - SP_ICON_HOST=${SP_ICON_HOST}
+# - SP_FLINK_DEBUG=${SP_FLINK_DEBUG}
+# ports:
+# - "8090:8090"
+
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
index 73ae870..c509a5e 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationController.java
@@ -16,97 +16,92 @@
package org.streampipes.processors.aggregation.flink.processor.aggregation;
-import org.streampipes.processors.aggregation.flink.config.AggregationFlinkConfig;
import org.streampipes.container.util.StandardTransportFormat;
import org.streampipes.model.DataProcessorType;
import org.streampipes.model.graph.DataProcessorDescription;
import org.streampipes.model.graph.DataProcessorInvocation;
import org.streampipes.model.schema.EventProperty;
import org.streampipes.model.schema.PropertyScope;
-import org.streampipes.model.util.SepaUtils;
+import org.streampipes.processors.aggregation.flink.config.AggregationFlinkConfig;
import org.streampipes.sdk.builder.ProcessingElementBuilder;
import org.streampipes.sdk.builder.StreamRequirementsBuilder;
import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.streampipes.sdk.helpers.*;
import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
import java.util.ArrayList;
import java.util.List;
public class AggregationController extends FlinkDataProcessorDeclarer<AggregationParameters> {
+ private static final String RESOURCE_ID = "strings";
+ private static final String PE_ID = "org.streampipes.processors.aggregation.flink.aggregation";
+
+ private static final String AGGREGATE_KEY = "aggregate";
+ private static final String AGGREGATED_VALUE_KEY = "aggregatedValue";
+ private static final String GROUP_BY_KEY = "groupBy";
+ private static final String OUTPUT_EVERY_KEY = "outputEvery";
+ private static final String TIME_WINDOW_KEY = "timeWindow";
+ private static final String OPERATION_KEY = "operation";
+
@Override
public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("aggregation", "Aggregation", "Performs different " +
- "aggregation functions")
+ return ProcessingElementBuilder.create(getLabel(PE_ID))
.category(DataProcessorType.AGGREGATE)
.iconUrl(AggregationFlinkConfig.iconBaseUrl + "/Aggregation_Icon_HQ.png")
.requiredStream(StreamRequirementsBuilder
.create()
- .requiredPropertyWithUnaryMapping(EpRequirements.numberReq(), Labels.from("aggregate",
- "Property Selection", "Specifies the event property from your stream that should be aggregated" +
- "."), PropertyScope.MEASUREMENT_PROPERTY)
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements.numberReq(),
+ getLabel(AGGREGATE_KEY),
+ PropertyScope.MEASUREMENT_PROPERTY)
.build())
- .naryMappingPropertyWithoutRequirement(Labels.from("groupBy", "Group by", "Partitions the incoming stream" +
- " by the selected event " +
- "properties"), PropertyScope.DIMENSION_PROPERTY)
- .outputStrategy(OutputStrategies.append(EpProperties.doubleEp(Labels.from("aggregated-value",
- "Aggregated Value", "The calculated aggregation value" ),
+ .naryMappingPropertyWithoutRequirement(
+ getLabel(GROUP_BY_KEY),
+ PropertyScope.DIMENSION_PROPERTY)
+ .outputStrategy(OutputStrategies.append(EpProperties.doubleEp(
+ getLabel(AGGREGATED_VALUE_KEY),
"aggregatedValue",
"http://schema.org/Number")))
- .requiredIntegerParameter("outputEvery", "Output Frequency", "Output values every " +
- "(seconds")
- .requiredIntegerParameter("timeWindow", "Time Window Size", "Size of the time window " +
- "in seconds")
- .requiredSingleValueSelection("operation", "Operation", "Aggregation operation type",
- Options.from("Average", "Sum", "Min", "Max"))
+ .requiredIntegerParameter(getLabel(OUTPUT_EVERY_KEY))
+ .requiredIntegerParameter(getLabel(TIME_WINDOW_KEY))
+ .requiredSingleValueSelection(getLabel(OPERATION_KEY),
+ Options.from(new Tuple2<>("Average", "AVG"),
+ new Tuple2<>("Sum", "SUM"),
+ new Tuple2<>("Min", "MIN"),
+ new Tuple2<>("Max", "MAX")))
.supportedFormats(StandardTransportFormat.standardFormat())
.supportedProtocols(StandardTransportFormat.standardProtocols())
.build();
}
+ private Label getLabel(String id) {
+ return Labels.fromResources(RESOURCE_ID, id);
+ }
+
@Override
public FlinkDataProcessorRuntime<AggregationParameters> getRuntime(DataProcessorInvocation graph,
ProcessingElementParameterExtractor extractor) {
- //List<String> groupBy = SepaUtils.getMultipleMappingPropertyNames(graph, "groupBy", true);
List<String> groupBy = new ArrayList<>();
- String aggregate = SepaUtils.getMappingPropertyName(graph, "aggregate");
-
- Integer outputEvery = extractor.singleValueParameter("outputEvery", Integer.class);
- Integer timeWindowSize = extractor.singleValueParameter("timeWindow", Integer.class);
- String aggregateOperation = extractor.selectedSingleValue("operation", String.class);
+ String aggregate = extractor.mappingPropertyValue(AGGREGATE_KEY);
+ Integer outputEvery = extractor.singleValueParameter(OUTPUT_EVERY_KEY, Integer.class);
+ Integer timeWindowSize = extractor.singleValueParameter(TIME_WINDOW_KEY, Integer.class);
+ String aggregateOperation = extractor.selectedSingleValueInternalName(OPERATION_KEY, String.class);
List<String> selectProperties = new ArrayList<>();
for (EventProperty p : graph.getInputStreams().get(0).getEventSchema().getEventProperties()) {
selectProperties.add(p.getRuntimeName());
}
- AggregationParameters staticParam = new AggregationParameters(graph, convert(aggregateOperation),
+ AggregationParameters staticParam = new AggregationParameters(graph, AggregationType.valueOf
+ (aggregateOperation),
outputEvery, groupBy,
aggregate, timeWindowSize, selectProperties);
- if (AggregationFlinkConfig.INSTANCE.getDebug()) {
- return new AggregationProgram(staticParam);
- } else {
- return new AggregationProgram(staticParam, new FlinkDeploymentConfig(AggregationFlinkConfig.JAR_FILE,
- AggregationFlinkConfig.INSTANCE.getFlinkHost(), AggregationFlinkConfig.INSTANCE.getFlinkPort()));
- }
+ return new AggregationProgram(staticParam, AggregationFlinkConfig.INSTANCE.getDebug());
}
-
- private AggregationType convert(String aggregateOperation) {
- if (aggregateOperation.equals("Average")) {
- return AggregationType.AVG;
- } else if (aggregateOperation.equals("Sum")) {
- return AggregationType.SUM;
- } else if (aggregateOperation.equals("Min")) {
- return AggregationType.MIN;
- } else {
- return AggregationType.MAX;
- }
- }
}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationProgram.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationProgram.java
index bfd7bb9..8a6dae0 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationProgram.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationProgram.java
@@ -21,20 +21,14 @@
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
-import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.processors.aggregation.flink.AbstractAggregationProgram;
import java.util.Map;
-public class AggregationProgram extends FlinkDataProcessorRuntime<AggregationParameters> {
+public class AggregationProgram extends AbstractAggregationProgram<AggregationParameters> {
- public AggregationProgram(AggregationParameters params) {
- super(params);
- setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
- }
-
- public AggregationProgram(AggregationParameters params, FlinkDeploymentConfig config) {
- super(params, config);
+ public AggregationProgram(AggregationParameters params, boolean debug) {
+ super(params, debug);
setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationType.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationType.java
index 5488685..590d21f 100644
--- a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationType.java
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationType.java
@@ -17,5 +17,8 @@
package org.streampipes.processors.aggregation.flink.processor.aggregation;
public enum AggregationType {
-AVG, MIN, MAX, SUM
+ AVG,
+ MIN,
+ MAX,
+ SUM
}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/aggregation.md b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/config/aggregation.md
similarity index 100%
rename from streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/aggregation.md
rename to streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/aggregation/config/aggregation.md
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountController.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountController.java
new file mode 100644
index 0000000..4f3eea1
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountController.java
@@ -0,0 +1,64 @@
+package org.streampipes.processors.aggregation.flink.processor.count;
+
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.streampipes.container.util.StandardTransportFormat;
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.aggregation.flink.config.AggregationFlinkConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class CountController extends FlinkDataProcessorDeclarer<CountParameters> {
+
+ private static final String TIME_WINDOW_KEY = "timeWindow";
+ private static final String SCALE_KEY = "scale";
+ private static final String COUNT_MAPPING = "count-mapping";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+
+ return ProcessingElementBuilder.create("org.streampipes.processors.aggregation.flink.count", "Count Aggregation",
+ "Performs an aggregation based on a given event property and outputs the number of occurrences.")
+ .category(DataProcessorType.AGGREGATE)
+ .iconUrl(AggregationFlinkConfig.iconBaseUrl + "/Counter_Icon_HQ.png")
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.anyProperty(), Labels.from(COUNT_MAPPING, "Field to count", "The field that contains the values which should be counted"), PropertyScope.DIMENSION_PROPERTY)
+ .build())
+ .outputStrategy(OutputStrategies.fixed(EpProperties.stringEp(Labels.empty(), "value", "http://schema.org/Text"), EpProperties.integerEp(Labels.empty(), "countValue",
+ "http://schema.org/Number")))
+ .requiredIntegerParameter(Labels.from(TIME_WINDOW_KEY, "Time Window Size", "Size of the time window " +
+ "in seconds"))
+ .requiredSingleValueSelection(Labels.from(SCALE_KEY, "Time Window Scale", ""),
+ Options.from(new Tuple2<>("Hours", "HOURS"),
+ new Tuple2<>("Minutes", "MINUTES"),
+ new Tuple2<>("Seconds", "SECONDS")))
+ .supportedFormats(StandardTransportFormat.standardFormat())
+ .supportedProtocols(StandardTransportFormat.standardProtocols())
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<CountParameters> getRuntime(DataProcessorInvocation graph,
+ ProcessingElementParameterExtractor extractor) {
+
+ Integer timeWindowSize = extractor.singleValueParameter(TIME_WINDOW_KEY, Integer.class);
+ String scale = extractor.selectedSingleValueInternalName(SCALE_KEY, String.class);
+ String fieldToCount = extractor.mappingPropertyValue(COUNT_MAPPING);
+
+ Time time = new TimeScale(scale).toFlinkTime(timeWindowSize);
+
+
+ CountParameters staticParam = new CountParameters(graph, time,
+ fieldToCount);
+
+ return new CountProgram(staticParam, AggregationFlinkConfig.INSTANCE.getDebug());
+
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountMapper.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountMapper.java
new file mode 100644
index 0000000..2daf0b5
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountMapper.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.aggregation.flink.processor.count;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+
+import java.util.Map;
+
+public class CountMapper implements MapFunction<Map<String, Object>, Tuple3<String, String, Integer>> {
+
+ private String fieldToCount;
+
+ public CountMapper(String fieldToCount) {
+ this.fieldToCount = fieldToCount;
+ }
+
+ @Override
+ public Tuple3<String, String, Integer> map(Map<String, Object> stringObjectMap) throws Exception {
+ return new Tuple3<>(fieldToCount, (String) stringObjectMap.get(fieldToCount), 1);
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountParameters.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountParameters.java
new file mode 100644
index 0000000..1e73696
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountParameters.java
@@ -0,0 +1,25 @@
+package org.streampipes.processors.aggregation.flink.processor.count;
+
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class CountParameters extends EventProcessorBindingParams {
+
+ private Time time;
+ private String fieldToCount;
+
+ public CountParameters(DataProcessorInvocation graph, Time time, String fieldToCount) {
+ super(graph);
+ this.time = time;
+ this.fieldToCount = fieldToCount;
+ }
+
+ public Time getTime() {
+ return time;
+ }
+
+ public String getFieldToCount() {
+ return fieldToCount;
+ }
+}
\ No newline at end of file
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountProgram.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountProgram.java
new file mode 100644
index 0000000..3bec6dd
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/CountProgram.java
@@ -0,0 +1,99 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.aggregation.flink.processor.count;
+
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.windowing.triggers.Trigger;
+import org.apache.flink.streaming.api.windowing.triggers.TriggerResult;
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.streampipes.processors.aggregation.flink.AbstractAggregationProgram;
+
+import java.util.Map;
+
+public class CountProgram extends AbstractAggregationProgram<CountParameters> {
+
+ public CountProgram(CountParameters params, boolean debug) {
+ super(params, debug);
+ setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... dataStreams) {
+ return dataStreams[0]
+ .map(new CountMapper(params.getFieldToCount()))
+ .keyBy(1)
+ .timeWindow(params.getTime())
+ .trigger(new Trigger<Tuple3<String, String, Integer>, TimeWindow>() {
+ @Override
+ public TriggerResult onElement(Tuple3<String, String, Integer> stringStringIntegerTuple3, long l, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+ return TriggerResult.FIRE;
+ }
+
+ @Override
+ public TriggerResult onProcessingTime(long l, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+ return TriggerResult.CONTINUE;
+ }
+
+ @Override
+ public TriggerResult onEventTime(long l, TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+ return TriggerResult.CONTINUE;
+ }
+
+ @Override
+ public void clear(TimeWindow timeWindow, TriggerContext triggerContext) throws Exception {
+
+ }
+ })
+ .sum(2)
+ .map(new Tuple2MapMapper());
+// .map()
+// .map(new MapFunction<Map<String, Object>, Map<String, Object>>() {
+// @Override
+// public Map<String, Object> map(Map<String, Object> stringObjectMap) throws Exception {
+// Map<String, Object> outMap = new HashMap<>();
+// outMap.put("value", params.getFieldToCount());
+// outMap.put("count", "");
+// return outMap;
+// }
+// });
+// .apply((WindowFunction<String, Map<String, Object>, Map<String, Object>, TimeWindow>) (map, timeWindow, iterable, collector) -> {
+// Integer count = 0;
+// Iterator<String> it = iterable.iterator();
+// while(it.hasNext()) {
+// String next = it.next();
+// if (next.equals(map.get(params.getFieldToCount()))) {
+// count++;
+// }
+// }
+//
+// Map<String, Object> outMap = new HashMap<>();
+// outMap.put("value", map.get(params.getFieldToCount()));
+// outMap.put("count", count);
+// collector.collect(outMap);
+// });
+
+
+ }
+
+ @Override
+ public void appendEnvironmentConfig(StreamExecutionEnvironment env) {
+ env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
+ }
+
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/TimeScale.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/TimeScale.java
new file mode 100644
index 0000000..06d1f20
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/TimeScale.java
@@ -0,0 +1,28 @@
+package org.streampipes.processors.aggregation.flink.processor.count;
+
+import org.apache.flink.streaming.api.windowing.time.Time;
+
+public class TimeScale {
+
+ private String value;
+
+ TimeScale(String value)
+ {
+ this.value = value;
+ }
+
+ public String value()
+ {
+ return value;
+ }
+
+ public Time toFlinkTime(Integer count) {
+ if (this.value.equals("minutes")) {
+ return Time.minutes(count);
+ } else if (this.value.equals("seconds")) {
+ return Time.seconds(count);
+ } else {
+ return Time.hours(count);
+ }
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/Tuple2MapMapper.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/Tuple2MapMapper.java
new file mode 100644
index 0000000..fa7abba
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/count/Tuple2MapMapper.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.aggregation.flink.processor.count;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.tuple.Tuple3;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class Tuple2MapMapper implements MapFunction<Tuple3<String, String, Integer>, Map<String, Object>> {
+ @Override
+ public Map<String, Object> map(Tuple3<String, String, Integer> in) throws Exception {
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("value", in.f1);
+ outMap.put("count", in.f2);
+ return outMap;
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRate.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRate.java
new file mode 100644
index 0000000..b9f3071
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRate.java
@@ -0,0 +1,37 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.aggregation.flink.processor.rate;
+
+import org.apache.flink.shaded.guava18.com.google.common.collect.Iterables;
+import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction;
+import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
+import org.apache.flink.util.Collector;
+
+import java.util.Map;
+
+public class EventRate implements AllWindowFunction<Map<String, Object>, Float, TimeWindow>{
+
+ private Integer timeWindowSize;
+
+ public EventRate(Integer timeWindowSize) {
+ this.timeWindowSize = timeWindowSize;
+ }
+
+ @Override
+ public void apply(TimeWindow timeWindow, Iterable<Map<String, Object>> iterable, Collector<Float> collector) throws Exception {
+ collector.collect((float) Iterables.size(iterable) / timeWindowSize);
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
new file mode 100644
index 0000000..78f9e30
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateController.java
@@ -0,0 +1,51 @@
+package org.streampipes.processors.aggregation.flink.processor.rate;
+
+import org.streampipes.container.util.StandardTransportFormat;
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.aggregation.flink.config.AggregationFlinkConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.EpProperties;
+import org.streampipes.sdk.helpers.EpRequirements;
+import org.streampipes.sdk.helpers.Labels;
+import org.streampipes.sdk.helpers.OutputStrategies;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class EventRateController extends FlinkDataProcessorDeclarer<EventRateParameter> {
+
+ private static final String RATE_KEY = "rate";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+
+ return ProcessingElementBuilder.create("org.streampipes.processor.aggregation.flink.rate", "Event rate", "Computes current event rate")
+ .category(DataProcessorType.AGGREGATE)
+ .iconUrl(AggregationFlinkConfig.getIconUrl("event_rate"))
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredProperty(EpRequirements.anyProperty())
+ .build())
+ .outputStrategy(OutputStrategies.fixed(EpProperties.doubleEp(Labels.empty(), "rate",
+ "http://schema.org/Number")))
+ .requiredIntegerParameter(Labels.from(RATE_KEY, "Time Baseline", "Time window size used for calculating the rate" +
+ "in seconds, also defines the output rate"))
+ .supportedFormats(StandardTransportFormat.standardFormat())
+ .supportedProtocols(StandardTransportFormat.standardProtocols())
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<EventRateParameter> getRuntime(DataProcessorInvocation graph,
+ ProcessingElementParameterExtractor extractor) {
+ Integer avgRate = extractor.singleValueParameter(RATE_KEY, Integer.class);
+
+ EventRateParameter staticParam = new EventRateParameter(graph, avgRate);
+
+ return new EventRateProgram(staticParam, AggregationFlinkConfig.INSTANCE.getDebug());
+
+ }
+}
\ No newline at end of file
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateParameter.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateParameter.java
new file mode 100644
index 0000000..623889f
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateParameter.java
@@ -0,0 +1,19 @@
+package org.streampipes.processors.aggregation.flink.processor.rate;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class EventRateParameter extends EventProcessorBindingParams {
+
+ private Integer avgRate;
+
+ public EventRateParameter(DataProcessorInvocation graph, int avgRate) {
+ super(graph);
+ this.avgRate = avgRate;
+ }
+
+ public int getAvgRate() {
+ return avgRate;
+ }
+
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateProgram.java b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateProgram.java
new file mode 100644
index 0000000..879d521
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/java/org/streampipes/processors/aggregation/flink/processor/rate/EventRateProgram.java
@@ -0,0 +1,50 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.aggregation.flink.processor.rate;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.util.Collector;
+import org.streampipes.processors.aggregation.flink.AbstractAggregationProgram;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class EventRateProgram extends AbstractAggregationProgram<EventRateParameter> {
+
+ public EventRateProgram(EventRateParameter params, boolean debug) {
+ super(params, debug);
+ setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... dataStreams) {
+ return dataStreams[0]
+ .timeWindowAll(Time.seconds(params.getAvgRate()))
+ .apply(new EventRate(params.getAvgRate()))
+ .flatMap(new FlatMapFunction<Float, Map<String, Object>>() {
+ @Override
+ public void flatMap(Float rate, Collector<Map<String, Object>> out) throws Exception {
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("rate", rate);
+ out.collect(outMap);
+ }
+ });
+ }
+
+}
diff --git a/streampipes-processors-aggregation-flink/src/main/resources/doc/aggregation.md b/streampipes-processors-aggregation-flink/src/main/resources/doc/aggregation.md
new file mode 100644
index 0000000..553bf49
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/resources/doc/aggregation.md
@@ -0,0 +1,5 @@
+****# Aggregation
+Performs different aggregation functions
+
+## Quality
+new SDK [ ]; tested [ ]
diff --git a/streampipes-processors-aggregation-flink/src/main/resources/strings b/streampipes-processors-aggregation-flink/src/main/resources/strings
new file mode 100644
index 0000000..38306fe
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/main/resources/strings
@@ -0,0 +1,21 @@
+org.streampipes.processors.aggregation.flink.aggregation.title=Aggregation
+org.streampipes.processors.aggregation.flink.aggregation.description=Performs different aggregation functions
+
+aggregate.title=Property Selection
+aggregate.description=Specifies the event property from your stream that should be aggregated.
+
+groupBy.title=Group by
+groupBy.description=Partitions the incoming stream by the selected event properties.
+
+outputEvery.title=Output Frequency
+outputEvery.description=Output values every (seconds)
+
+timeWindow.title=Time Window Size
+timeWindow.description=Size of the time window in seconds
+
+operation.title=Operation
+operation.description=Aggregation operation type
+
+aggregatedValue.title=Aggregated value
+aggregatedValue.description=The calculated aggregation value
+
diff --git a/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationTestData.java b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationTestData.java
new file mode 100644
index 0000000..41b4922
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/aggregation/AggregationTestData.java
@@ -0,0 +1,65 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.aggregation.flink.processor.aggregation;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AggregationTestData {
+
+ private List<Map<String, Object>> expectedOutput;
+ private List<Map<String, Object>> input;
+
+ public AggregationTestData() {
+ buildOutput();
+ buildInput();
+ }
+
+ private void buildOutput() {
+ this.expectedOutput = new ArrayList<>();
+ this.expectedOutput.add(buildOutputMap(1, 1.0));
+ this.expectedOutput.add(buildOutputMap(2, 1.5));
+ }
+
+ private void buildInput() {
+ this.input = new ArrayList<>();
+ input.add(buildMap(1));
+ input.add(buildMap(2));
+ }
+
+ private Map<String, Object> buildOutputMap(Object value, Object aggregatedValue) {
+ Map<String, Object> map = buildMap(value);
+ map.put("aggregatedValue", aggregatedValue);
+ return map;
+ }
+
+ private Map<String, Object> buildMap(Object value) {
+ Map<String, Object> map = new HashMap<>();
+ map.put("sensorId", "a");
+ map.put("value", value);
+ return map;
+ }
+
+ public List<Map<String, Object>> getExpectedOutput() {
+ return expectedOutput;
+ }
+
+ public List<Map<String, Object>> getInput() {
+ return input;
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/aggregation/TestAggregationProgram.java b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/aggregation/TestAggregationProgram.java
new file mode 100644
index 0000000..8f6baab
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/aggregation/TestAggregationProgram.java
@@ -0,0 +1,70 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.aggregation.flink.processor.aggregation;
+
+import io.flinkspector.core.collection.ExpectedRecords;
+import io.flinkspector.datastream.DataStreamTestBase;
+import io.flinkspector.datastream.input.EventTimeInput;
+import io.flinkspector.datastream.input.EventTimeInputBuilder;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.junit.Test;
+import org.streampipes.test.generator.InvocationGraphGenerator;
+
+import java.util.*;
+
+//@RunWith(Parameterized.class)
+public class TestAggregationProgram extends DataStreamTestBase {
+
+// @Parameterized.Parameters
+// public static Iterable<Object[]> algorithm() {
+// return Arrays.asList(new Object[][]{
+// {"a", 1},
+// {new Sha1HashAlgorithm(), HashAlgorithmType.SHA1},
+// {new Sha2HashAlgorithm(), HashAlgorithmType.SHA2}
+// });
+// }
+
+
+ @Test
+ public void testAggregationProgram() {
+ AggregationParameters params = makeParams();
+ AggregationProgram program = new AggregationProgram(params, true);
+ AggregationTestData testData = new AggregationTestData();
+
+ DataStream<Map<String, Object>> stream = program.getApplicationLogic(createTestStream(makeInputData(testData)));
+
+ ExpectedRecords<Map<String, Object>> expected =
+ new ExpectedRecords<Map<String, Object>>().expectAll(testData.getExpectedOutput());
+
+ assertStream(stream, expected);
+ }
+
+ private AggregationParameters makeParams() {
+ return new AggregationParameters(InvocationGraphGenerator.makeEmptyInvocation(new AggregationController().declareModel()),
+ AggregationType.AVG,
+ 1,
+ Arrays.asList("sensorId"),
+ "value",
+ 10,
+ Arrays.asList("value"));
+ }
+
+ private EventTimeInput<Map<String, Object>> makeInputData(AggregationTestData testData) {
+ return EventTimeInputBuilder.startWith(testData.getInput().get(0))
+ .emit(testData.getInput().get(1), after(1, seconds));
+ }
+
+}
diff --git a/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/count/TestCountProgram.java b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/count/TestCountProgram.java
new file mode 100644
index 0000000..91b7ac7
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/count/TestCountProgram.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.aggregation.flink.processor.count;
+
+import io.flinkspector.core.collection.ExpectedRecords;
+import io.flinkspector.datastream.DataStreamTestBase;
+import io.flinkspector.datastream.input.EventTimeInput;
+import io.flinkspector.datastream.input.EventTimeInputBuilder;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.junit.Test;
+import org.streampipes.test.generator.InvocationGraphGenerator;
+
+import java.util.*;
+
+public class TestCountProgram extends DataStreamTestBase {
+
+ @Test
+ public void testCountProgram() {
+
+ EventTimeInput input = makeInputData(makeTestData(), makeTestData().size());
+
+ ExpectedRecords<Map<String, Object>> expected =
+ new ExpectedRecords<Map<String, Object>>().expectAll(getOutput());
+
+ runProgram(input, expected);
+ }
+
+ @Test
+ public void testOutOfWindow() {
+
+ EventTimeInput input = makeInputData(makeTestData(), 2);
+
+ ExpectedRecords<Map<String, Object>> expected =
+ new ExpectedRecords<Map<String, Object>>().expectAll(getOutOfWindowOutput());
+
+ runProgram(input, expected);
+ }
+
+ private void runProgram(EventTimeInput<Map<String, Object>> input, ExpectedRecords<Map<String, Object>> expected) {
+ CountParameters params = new CountParameters(InvocationGraphGenerator.makeEmptyInvocation(new CountController().declareModel()), Time.seconds(10), "field");
+
+ CountProgram program = new CountProgram(params, true);
+
+ DataStream<Map<String, Object>> stream = program.getApplicationLogic(createTestStream(input));
+
+ assertStream(stream, expected);
+ }
+
+ private Collection<Map<String, Object>> getOutput() {
+ List<Map<String, Object>> outRecords = new ArrayList<>();
+ outRecords.add(makeOutMap("v1", 1));
+ outRecords.add(makeOutMap("v2", 1));
+ outRecords.add(makeOutMap("v1", 2));
+ outRecords.add(makeOutMap("v3", 1));
+ outRecords.add(makeOutMap("v2", 2));
+
+ return outRecords;
+ }
+
+ private Collection<Map<String, Object>> getOutOfWindowOutput() {
+ List<Map<String, Object>> outRecords = new ArrayList<>();
+ outRecords.add(makeOutMap("v1", 1));
+ outRecords.add(makeOutMap("v2", 1));
+ outRecords.add(makeOutMap("v1", 1));
+ outRecords.add(makeOutMap("v3", 1));
+ outRecords.add(makeOutMap("v2", 1));
+
+ return outRecords;
+ }
+
+ private Map<String, Object> makeOutMap(String key, Integer count) {
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("value", key);
+ outMap.put("count", count);
+ return outMap;
+ }
+
+ private EventTimeInput<Map<String, Object>> makeInputData(List<Map<String, Object>> testData, Integer splitIndex) {
+ EventTimeInputBuilder<Map<String, Object>> builder = EventTimeInputBuilder.startWith(testData.get(0));
+
+ for (int i = 1; i < splitIndex; i++) {
+ builder.emit(testData.get(i), after(1, seconds));
+ }
+
+ for (int j = splitIndex; j < testData.size(); j++) {
+ builder.emit(testData.get(j), after(10, seconds));
+ }
+
+ return builder;
+ }
+
+ private List<Map<String, Object>> makeTestData() {
+ List<Map<String, Object>> inMap = new ArrayList<>();
+ inMap.add(makeMap("v1"));
+ inMap.add(makeMap("v2"));
+ inMap.add(makeMap("v1"));
+ inMap.add(makeMap("v3"));
+ inMap.add(makeMap("v2"));
+
+ return inMap;
+ }
+
+ private Map<String, Object> makeMap(String s) {
+ Map<String, Object> testMap = new HashMap<>();
+ testMap.put("field", s);
+ return testMap;
+ }
+}
diff --git a/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/rate/TestRateProgram.java b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/rate/TestRateProgram.java
new file mode 100644
index 0000000..ce1aa14
--- /dev/null
+++ b/streampipes-processors-aggregation-flink/src/test/java/org/streampipes/processors/aggregation/flink/processor/rate/TestRateProgram.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.aggregation.flink.processor.rate;
+
+import io.flinkspector.core.collection.ExpectedRecords;
+import io.flinkspector.datastream.DataStreamTestBase;
+import io.flinkspector.datastream.input.EventTimeInput;
+import io.flinkspector.datastream.input.EventTimeInputBuilder;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.streampipes.test.generator.InvocationGraphGenerator;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+
+@RunWith(Parameterized.class)
+public class TestRateProgram extends DataStreamTestBase {
+
+ @Parameterized.Parameters
+ public static Iterable<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {1, 1000, TimeUnit.MILLISECONDS, 1.0f, 1},
+ {10, 1000, TimeUnit.MILLISECONDS, 1.0f, 1},
+ {100, 1000, TimeUnit.MILLISECONDS, 1.0f, 1},
+ {10, 100, TimeUnit.MILLISECONDS, 1.0f, 10},
+ {2, 500, TimeUnit.MILLISECONDS, 2.0f, 1},
+ {4, 250, TimeUnit.MILLISECONDS, 4.0f, 1},
+ {8, 250, TimeUnit.MILLISECONDS, 4.0f, 2},
+ });
+ }
+
+ @Parameterized.Parameter
+ public Integer numEvents;
+
+ @Parameterized.Parameter(1)
+ public Integer waitTime;
+
+ @Parameterized.Parameter(2)
+ public TimeUnit timeUnit;
+
+ @Parameterized.Parameter(3)
+ public Float expectedFrequency;
+
+ @Parameterized.Parameter(4)
+ public Integer timeWindowSize;
+
+ @Test
+ public void testRateProgram() {
+ EventRateParameter params = new EventRateParameter(InvocationGraphGenerator.makeEmptyInvocation(new EventRateController().declareModel()), timeWindowSize);
+
+ EventRateProgram program = new EventRateProgram(params, true);
+
+ DataStream<Map<String, Object>> stream = program.getApplicationLogic(createTestStream(makeInputData(numEvents, waitTime, timeUnit)));
+
+ ExpectedRecords<Map<String, Object>> expected =
+ new ExpectedRecords<Map<String, Object>>().expectAll(getOutput(timeWindowSize, expectedFrequency, numEvents));
+
+ assertStream(stream, expected);
+ }
+
+ private Collection<Map<String, Object>> getOutput(Integer timeWindowSize, Float eventsPerSecond, Integer numEvents) {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("rate", eventsPerSecond);
+
+ for (int i = 0; i < numEvents % timeWindowSize; i++) {
+ allEvents.add(outMap);
+ }
+
+ return allEvents;
+ }
+
+ private EventTimeInput<Map<String, Object>> makeInputData(Integer count, Integer time, TimeUnit timeUnit) {
+ List<Map<String, Object>> testData = makeTestData(count);
+ EventTimeInputBuilder<Map<String, Object>> builder = EventTimeInputBuilder.startWith(testData.get(0));
+
+ for (int i = 1; i < testData.size(); i++) {
+ builder.emit(testData.get(i), after(time, timeUnit));
+ }
+
+ return builder;
+ }
+
+ private List<Map<String, Object>> makeTestData(Integer count) {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+ Map<String, Object> event = new HashMap<>();
+ event.put("test", 1);
+
+ for (int i = 0; i < count; i++) {
+ allEvents.add(event);
+ }
+
+ return allEvents;
+ }
+
+
+}
diff --git a/streampipes-processors-enricher-flink/Dockerfile b/streampipes-processors-enricher-flink/Dockerfile
index b5eb662..f47bb3f 100644
--- a/streampipes-processors-enricher-flink/Dockerfile
+++ b/streampipes-processors-enricher-flink/Dockerfile
@@ -3,6 +3,6 @@
EXPOSE 8090
ENV CONSUL_LOCATION consul
-ADD ./target/streampipes-processors-enricher-flink.jar /streampipes-processors-enricher-flink.jar
+ADD ./target/streampipes-processors-enricher-flink.jar /streampipes-processing-element-container.jar
-ENTRYPOINT ["java", "-jar", "/streampipes-processors-enricher-flink.jar"]
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-enricher-flink/deployment/docker-compose.yml b/streampipes-processors-enricher-flink/deployment/docker-compose.yml
new file mode 100644
index 0000000..9ceda8b
--- /dev/null
+++ b/streampipes-processors-enricher-flink/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-enricher-flink:
+ image: ${SP_PE_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-enricher-flink:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-enricher-flink/development/.env b/streampipes-processors-enricher-flink/development/.env
new file mode 100644
index 0000000..b5ab2b6
--- /dev/null
+++ b/streampipes-processors-enricher-flink/development/.env
@@ -0,0 +1,8 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6010
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_ELASTICSEARCH_HOST=localhost
+SP_FLINK_DEBUG=true
\ No newline at end of file
diff --git a/streampipes-processors-enricher-flink/pom.xml b/streampipes-processors-enricher-flink/pom.xml
index 5757675..fc2eca2 100644
--- a/streampipes-processors-enricher-flink/pom.xml
+++ b/streampipes-processors-enricher-flink/pom.xml
@@ -1,11 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>streampipes-pipeline-elements</artifactId>
<groupId>org.streampipes</groupId>
- <version>0.55.3-SNAPSHOT</version>
+ <version>0.60.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -13,7 +11,6 @@
<properties>
- <streampipes.version>0.55.3-SNAPSHOT</streampipes.version>
<elasticsearch.version>5.2.2</elasticsearch.version>
</properties>
@@ -21,7 +18,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-container-standalone</artifactId>
- <version>${streampipes.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.kafka</groupId>
@@ -36,7 +32,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-commons</artifactId>
- <version>${streampipes.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.kafka</groupId>
@@ -51,7 +46,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-wrapper-flink</artifactId>
- <version>${streampipes.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
@@ -71,33 +65,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-sdk</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-filesystem_2.11</artifactId>
- <version>1.4.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>3.6.1</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.2.3</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>6.2.3</version>
- <exclusions>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@@ -114,56 +81,12 @@
<artifactId>streampipes-config</artifactId>
<version>${streampipes.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-cep-scala_2.11</artifactId>
- <version>1.4.0</version>
- </dependency>
</dependencies>
<build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.0</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.10.4</version>
- </plugin>
- </plugins>
- </pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
@@ -201,85 +124,8 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>external.atlassian.jgitflow</groupId>
- <artifactId>jgitflow-maven-plugin</artifactId>
- <version>1.0-m5.1</version>
- <configuration>
- <flowInitContext>
- <masterBranchName>master</masterBranchName>
- <developBranchName>dev</developBranchName>
- <featureBranchPrefix>feature-</featureBranchPrefix>
- <releaseBranchPrefix>release-</releaseBranchPrefix>
- <hotfixBranchPrefix>hotfix-</hotfixBranchPrefix>
- <versionTagPrefix>version-</versionTagPrefix>
- </flowInitContext>
- <noDeploy>false</noDeploy>
- <autoVersionSubmodules>true</autoVersionSubmodules>
- <pushReleases>false</pushReleases>
- <localOnly>true</localOnly>
- <squash>false</squash>
- <scmCommentPrefix>[RELEASE] [skip-ci]</scmCommentPrefix>
- <enableSshAgent>true</enableSshAgent>
- </configuration>
- </plugin>
</plugins>
<finalName>streampipes-processors-enricher-flink</finalName>
</build>
-
- <scm>
- <developerConnection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-flink.git
- </developerConnection>
- <connection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-flink.git</connection>
- <url>https://github.com/streampipes/pe-examples-flink</url>
- </scm>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
-
- <developers>
- <developer>
- <name>Dominik Riemer</name>
- <email>riemer@fzi.de</email>
- </developer>
- <developer>
- <name>Philipp Zehnder</name>
- <email>zehnder@fzi.de</email>
- </developer>
- </developers>
-
- <repositories>
- <repository>
- <id>laus</id>
- <name>nexus repository</name>
- <url>https://laus.fzi.de/nexus/content/repositories/public/</url>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </snapshots>
- </repository>
- </repositories>
-
- <distributionManagement>
- <repository>
- <id>sonatype</id>
- <name>Releases</name>
- <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
- </repository>
- <snapshotRepository>
- <id>deployment</id>
- <name>Internal Releases</name>
- <url>https://laus.fzi.de/nexus/content/repositories/snapshots/</url>
- </snapshotRepository>
- </distributionManagement>
</project>
\ No newline at end of file
diff --git a/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/AbstractEnricherProgram.java b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/AbstractEnricherProgram.java
new file mode 100644
index 0000000..5c6a530
--- /dev/null
+++ b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/AbstractEnricherProgram.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.enricher.flink;
+
+import org.streampipes.processors.enricher.flink.config.EnricherFlinkConfig;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public abstract class AbstractEnricherProgram<B extends EventProcessorBindingParams> extends FlinkDataProcessorRuntime<B> {
+
+ public AbstractEnricherProgram(B params, boolean debug) {
+ super(params, debug);
+ }
+
+ public AbstractEnricherProgram(B params) {
+ super(params, false);
+ }
+
+ @Override
+ protected FlinkDeploymentConfig getDeploymentConfig() {
+ return new FlinkDeploymentConfig(EnricherFlinkConfig.JAR_FILE,
+ EnricherFlinkConfig.INSTANCE.getFlinkHost(), EnricherFlinkConfig.INSTANCE.getFlinkPort());
+ }
+
+}
+
diff --git a/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
index d638eb2..492f7e2 100644
--- a/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
+++ b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/config/EnricherFlinkConfig.java
@@ -23,7 +23,7 @@
INSTANCE;
private SpConfig config;
- public static final String JAR_FILE = "./streampipes-examples-flink.jar";
+ public static final String JAR_FILE = "./streampipes-processing-element-container.jar";
private final static String service_id = "pe/org.streampipes.processors.enricher.flink";
private final static String service_name = "Processors Enricher Flink";
diff --git a/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampController.java b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampController.java
index a7dee62..1a53828 100644
--- a/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampController.java
+++ b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampController.java
@@ -16,16 +16,17 @@
package org.streampipes.processors.enricher.flink.processor.timestamp;
+import org.streampipes.container.util.StandardTransportFormat;
import org.streampipes.model.graph.DataProcessorDescription;
import org.streampipes.model.graph.DataProcessorInvocation;
import org.streampipes.processors.enricher.flink.config.EnricherFlinkConfig;
import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
import org.streampipes.sdk.helpers.*;
import org.streampipes.vocabulary.SO;
import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
public class TimestampController extends FlinkDataProcessorDeclarer<TimestampParameters> {
@@ -33,13 +34,17 @@
@Override
public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("enrich_configurable_timestamp", "Configurable Flink Timestamp Enrichment",
- "Appends the current time in ms to the event payload using Flink")
+ return ProcessingElementBuilder.create("org.streampipes.processors.enricher.flink" +
+ ".enrich-timestamp", "Timestamp Enricher",
+ "Appends the current time in ms to the event payload")
.iconUrl(EnricherFlinkConfig.getIconUrl("enrich-timestamp-icon"))
- .requiredPropertyStream1(EpRequirements.anyProperty())
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredProperty(EpRequirements.anyProperty())
+ .build())
.outputStrategy(OutputStrategies.append(
EpProperties.longEp(Labels.empty(), APPEND_PROPERTY, SO.DateTime)))
- .supportedProtocols(SupportedProtocols.kafka())
+ .supportedProtocols(StandardTransportFormat.standardProtocols())
.supportedFormats(SupportedFormats.jsonFormat())
.build();
}
@@ -49,16 +54,12 @@
DataProcessorInvocation graph,
ProcessingElementParameterExtractor extractor) {
- TimestampParameters staticParam = new TimestampParameters(
+ TimestampParameters staticParam = new TimestampParameters(
graph,
APPEND_PROPERTY);
- if (EnricherFlinkConfig.INSTANCE.getDebug()) {
- return new TimestampProgram(staticParam);
- } else {
- return new TimestampProgram(staticParam, new FlinkDeploymentConfig(EnricherFlinkConfig.JAR_FILE,
- EnricherFlinkConfig.INSTANCE.getFlinkHost(), EnricherFlinkConfig.INSTANCE.getFlinkPort()));
- }
+
+ return new TimestampProgram(staticParam, EnricherFlinkConfig.INSTANCE.getDebug());
}
}
diff --git a/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampProgram.java b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampProgram.java
index 33812af..2d73a2c 100644
--- a/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampProgram.java
+++ b/streampipes-processors-enricher-flink/src/main/java/org/streampipes/processors/enricher/flink/processor/timestamp/TimestampProgram.java
@@ -17,25 +17,20 @@
package org.streampipes.processors.enricher.flink.processor.timestamp;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.processors.enricher.flink.AbstractEnricherProgram;
import java.util.Map;
-public class TimestampProgram extends FlinkDataProcessorRuntime<TimestampParameters> {
+public class TimestampProgram extends AbstractEnricherProgram<TimestampParameters> {
- public TimestampProgram(TimestampParameters params) {
- super(params);
- }
-
- public TimestampProgram(TimestampParameters params, FlinkDeploymentConfig config) {
- super(params, config);
+ public TimestampProgram(TimestampParameters params, boolean debug) {
+ super(params, debug);
}
@Override
protected DataStream<Map<String, Object>> getApplicationLogic(
DataStream<Map<String, Object>>... messageStream) {
- return (DataStream<Map<String, Object>>) messageStream[0]
+ return messageStream[0]
.flatMap(new TimestampEnricher(params.getAppendTimePropertyName()));
}
diff --git a/streampipes-processors-filters-jvm/Dockerfile b/streampipes-processors-filters-jvm/Dockerfile
index ba2641c..1675b39 100644
--- a/streampipes-processors-filters-jvm/Dockerfile
+++ b/streampipes-processors-filters-jvm/Dockerfile
@@ -3,6 +3,6 @@
EXPOSE 8090
ENV CONSUL_LOCATION consul
-ADD ./target/streampipes-processors-filters-jvm.jar /streampipes-processors-filters-jvm.jar
+ADD ./target/streampipes-processors-filters-jvm.jar /streampipes-processing-element-container.jar
-ENTRYPOINT ["java", "-jar", "/streampipes-processors-filters-jvm.jar"]
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-filters-jvm/deployment/docker-compose.yml b/streampipes-processors-filters-jvm/deployment/docker-compose.yml
new file mode 100644
index 0000000..9f18687
--- /dev/null
+++ b/streampipes-processors-filters-jvm/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-filters-jvm:
+ image: ${SP_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-filters-jvm:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-filters-jvm/development/.env b/streampipes-processors-filters-jvm/development/.env
new file mode 100644
index 0000000..b7f86f5
--- /dev/null
+++ b/streampipes-processors-filters-jvm/development/.env
@@ -0,0 +1,10 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6015
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_COUCHDB_HOST=localhost
+SP_JMS_HOST=localhost
+SP_NGINX_HOST=localhost
+SP_NGINX_PORT=8082
\ No newline at end of file
diff --git a/streampipes-processors-filters-jvm/pom.xml b/streampipes-processors-filters-jvm/pom.xml
index 553fe1d..9a4bf1a 100644
--- a/streampipes-processors-filters-jvm/pom.xml
+++ b/streampipes-processors-filters-jvm/pom.xml
@@ -1,96 +1,45 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>streampipes-pipeline-elements</artifactId>
<groupId>org.streampipes</groupId>
- <version>0.55.3-SNAPSHOT</version>
+ <version>0.60.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>streampipes-processors-filters-jvm</artifactId>
- <properties>
- <streampipes.version>0.55.3-SNAPSHOT</streampipes.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-wrapper-standalone</artifactId>
- <version>${streampipes.version}</version>
</dependency>
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-sdk</artifactId>
- <version>${streampipes.version}</version>
</dependency>
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-container-standalone</artifactId>
- <version>${streampipes.version}</version>
</dependency>
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-config</artifactId>
- <version>${streampipes.version}</version>
</dependency>
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-dataformat-json</artifactId>
- <version>${streampipes.version}</version>
</dependency>
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-messaging-kafka</artifactId>
- <version>${streampipes.version}</version>
</dependency>
</dependencies>
<build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.10.4</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.0</version>
- </plugin>
- </plugins>
- </pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
@@ -125,107 +74,8 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>external.atlassian.jgitflow</groupId>
- <artifactId>jgitflow-maven-plugin</artifactId>
- <version>1.0-m5.1</version>
- <configuration>
- <flowInitContext>
- <masterBranchName>master</masterBranchName>
- <developBranchName>dev</developBranchName>
- <featureBranchPrefix>feature-</featureBranchPrefix>
- <releaseBranchPrefix>release-</releaseBranchPrefix>
- <hotfixBranchPrefix>hotfix-</hotfixBranchPrefix>
- <versionTagPrefix>version-</versionTagPrefix>
- </flowInitContext>
- <noDeploy>false</noDeploy>
- <autoVersionSubmodules>true</autoVersionSubmodules>
- <pushReleases>false</pushReleases>
- <localOnly>true</localOnly>
- <squash>false</squash>
- <scmCommentPrefix>[RELEASE] [skip-ci]</scmCommentPrefix>
- <enableSshAgent>true</enableSshAgent>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-gpg-plugin</artifactId>
- <version>1.6</version>
- <executions>
- <execution>
- <id>sign-artifacts</id>
- <phase>verify</phase>
- <goals>
- <goal>sign</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <gpgArguments>
- <arg>--pinentry-mode</arg>
- <arg>loopback</arg>
- </gpgArguments>
- </configuration>
- </plugin>
</plugins>
<finalName>streampipes-processors-filters-jvm</finalName>
</build>
-
- <scm>
- <developerConnection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-jvm.git
- </developerConnection>
- <connection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-jvm.git</connection>
- <url>https://github.com/streampipes/pe-examples-jvm</url>
- </scm>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
-
- <developers>
- <developer>
- <name>Dominik Riemer</name>
- <email>riemer@fzi.de</email>
- </developer>
- <developer>
- <name>Philipp Zehnder</name>
- <email>zehnder@fzi.de</email>
- </developer>
- </developers>
-
- <repositories>
- <repository>
- <id>laus</id>
- <name>nexus repository</name>
- <url>https://laus.fzi.de/nexus/content/repositories/public/</url>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </snapshots>
- </repository>
- </repositories>
-
- <distributionManagement>
- <repository>
- <id>sonatype</id>
- <name>Releases</name>
- <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
- </repository>
- <snapshotRepository>
- <id>deployment</id>
- <name>Internal Releases</name>
- <url>https://laus.fzi.de/nexus/content/repositories/snapshots/</url>
- </snapshotRepository>
- </distributionManagement>
-
-
</project>
\ No newline at end of file
diff --git a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/FiltersJvmInit.java b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/FiltersJvmInit.java
index da371bf..84bab6a 100644
--- a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/FiltersJvmInit.java
+++ b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/FiltersJvmInit.java
@@ -19,8 +19,10 @@
import org.streampipes.container.init.DeclarersSingleton;
import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
import org.streampipes.dataformat.json.JsonDataFormatFactory;
+import org.streampipes.messaging.jms.SpJmsProtocolFactory;
import org.streampipes.processors.filters.jvm.config.FiltersJvmConfig;
import org.streampipes.processors.filters.jvm.processor.numericalfilter.NumericalFilterController;
+import org.streampipes.processors.filters.jvm.processor.projection.ProjectionController;
import org.streampipes.processors.filters.jvm.processor.textfilter.TextFilterController;
import org.streampipes.messaging.kafka.SpKafkaProtocolFactory;
@@ -31,10 +33,12 @@
DeclarersSingleton
.getInstance()
.add(new NumericalFilterController())
- .add(new TextFilterController());
+ .add(new TextFilterController())
+ .add(new ProjectionController());
DeclarersSingleton.getInstance().registerDataFormat(new JsonDataFormatFactory());
DeclarersSingleton.getInstance().registerProtocol(new SpKafkaProtocolFactory());
+ DeclarersSingleton.getInstance().registerProtocol(new SpJmsProtocolFactory());
new FiltersJvmInit().init(FiltersJvmConfig.INSTANCE);
}
diff --git a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/numericalfilter/NumericalFilterController.java b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/numericalfilter/NumericalFilterController.java
index 82e2c3a..a806262 100644
--- a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/numericalfilter/NumericalFilterController.java
+++ b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/numericalfilter/NumericalFilterController.java
@@ -20,36 +20,35 @@
import org.streampipes.model.graph.DataProcessorDescription;
import org.streampipes.model.graph.DataProcessorInvocation;
import org.streampipes.model.schema.PropertyScope;
-import org.streampipes.model.util.SepaUtils;
import org.streampipes.processors.filters.jvm.config.FiltersJvmConfig;
import org.streampipes.sdk.builder.ProcessingElementBuilder;
import org.streampipes.sdk.builder.StreamRequirementsBuilder;
import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
-import org.streampipes.sdk.helpers.EpRequirements;
-import org.streampipes.sdk.helpers.Labels;
-import org.streampipes.sdk.helpers.Options;
-import org.streampipes.sdk.helpers.OutputStrategies;
-import org.streampipes.sdk.helpers.SupportedFormats;
-import org.streampipes.sdk.helpers.SupportedProtocols;
+import org.streampipes.sdk.helpers.*;
import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
public class NumericalFilterController extends StandaloneEventProcessingDeclarer<NumericalFilterParameters> {
+ private static final String NUMBER_MAPPING = "number-mapping";
+ private static final String VALUE = "value";
+ private static final String OPERATION = "operation";
+
@Override
public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("numericalfilter", "Numerical Filter", "Numerical Filter Description")
+ return ProcessingElementBuilder.create("org.streampipes.processors.filters.jvm.numericalfilter", "Numerical Filter", "Numerical Filter Description")
.category(DataProcessorType.FILTER)
.iconUrl(FiltersJvmConfig.getIconUrl("Numerical_Filter_Icon_HQ"))
- .requiredStream(StreamRequirementsBuilder.create().requiredPropertyWithUnaryMapping(EpRequirements
- .numberReq(), Labels.from("number", "Specifies the field name where the filter operation should" +
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.numberReq(), Labels.from(NUMBER_MAPPING, "Specifies the field name where the filter operation should" +
" be applied " +
"on.", ""), PropertyScope.NONE).build())
.outputStrategy(OutputStrategies.keep())
- .requiredSingleValueSelection("operation", "Filter Operation", "Specifies the filter " +
- "operation that should be applied on the field", Options.from("<", "<=", ">", ">=", "=="))
- .requiredFloatParameter("value", "Threshold value", "Specifies a threshold value.", "number")
- .supportedProtocols(SupportedProtocols.kafka())
+ .requiredSingleValueSelection(Labels.from(OPERATION, "Filter Operation", "Specifies the filter " +
+ "operation that should be applied on the field"), Options.from("<", "<=", ">", ">=", "=="))
+ .requiredFloatParameter(Labels.from(VALUE, "Threshold value", "Specifies a threshold value."), "number")
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
.supportedFormats(SupportedFormats.jsonFormat())
.build();
@@ -60,8 +59,8 @@
(DataProcessorInvocation sepa) {
ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(sepa);
- Double threshold = extractor.singleValueParameter("value", Double.class);
- String stringOperation = extractor.selectedSingleValue("operation", String.class);
+ Double threshold = extractor.singleValueParameter(VALUE, Double.class);
+ String stringOperation = extractor.selectedSingleValue(OPERATION, String.class);
String operation = "GT";
@@ -75,8 +74,7 @@
operation = "EQ";
}
- String filterProperty = SepaUtils.getMappingPropertyName(sepa,
- "number", true);
+ String filterProperty = extractor.mappingPropertyValue(NUMBER_MAPPING);
NumericalFilterParameters staticParam = new NumericalFilterParameters(sepa, threshold, NumericalOperator.valueOf
(operation)
diff --git a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/Projection.java b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/Projection.java
new file mode 100644
index 0000000..3260551
--- /dev/null
+++ b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/Projection.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.filters.jvm.processor.projection;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Projection extends StandaloneEventProcessorEngine<ProjectionParameters> {
+
+ private List<String> outputKeys;
+
+ public Projection(ProjectionParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(ProjectionParameters projectionParameters, DataProcessorInvocation dataProcessorInvocation) {
+ this.outputKeys = projectionParameters.getOutputKeys();
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> in, String sourceInfo, SpOutputCollector out) {
+ Map<String, Object> outEvent = new HashMap<>();
+ for(String outputKey : outputKeys) {
+ outEvent.put(outputKey, in.get(outputKey));
+ }
+ out.onEvent(outEvent);
+ }
+
+ @Override
+ public void onDetach() {
+
+ }
+}
diff --git a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/ProjectionController.java b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/ProjectionController.java
new file mode 100644
index 0000000..bf1fa2e
--- /dev/null
+++ b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/ProjectionController.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.filters.jvm.processor.projection;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.EventProperty;
+import org.streampipes.processors.filters.jvm.config.FiltersJvmConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.helpers.EpRequirements;
+import org.streampipes.sdk.helpers.OutputStrategies;
+import org.streampipes.sdk.helpers.SupportedFormats;
+import org.streampipes.sdk.helpers.SupportedProtocols;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class ProjectionController extends StandaloneEventProcessingDeclarer<ProjectionParameters> {
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.filters.jvm.project", "Projection", "Outputs a selectable subset of an input event type")
+ .category(DataProcessorType.TRANSFORM)
+ .iconUrl(FiltersJvmConfig.getIconUrl("projection"))
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredProperty(EpRequirements.anyProperty())
+ .build())
+ .outputStrategy(OutputStrategies.custom())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .supportedProtocols(SupportedProtocols.jms(), SupportedProtocols.kafka())
+ .build();
+ }
+
+ @Override
+ public ConfiguredEventProcessor<ProjectionParameters>
+ onInvocation(DataProcessorInvocation graph) {
+
+ List<String> outputKeys = graph
+ .getOutputStream()
+ .getEventSchema()
+ .getEventProperties()
+ .stream()
+ .map(EventProperty::getRuntimeName)
+ .collect(Collectors.toList());
+
+ ProjectionParameters staticParam = new ProjectionParameters(
+ graph, outputKeys);
+
+ return new ConfiguredEventProcessor<>(staticParam, () -> new Projection(staticParam));
+ }
+}
diff --git a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/ProjectionParameters.java b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/ProjectionParameters.java
new file mode 100644
index 0000000..437a19f
--- /dev/null
+++ b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/projection/ProjectionParameters.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.filters.jvm.processor.projection;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+import java.util.List;
+
+public class ProjectionParameters extends EventProcessorBindingParams {
+
+ private List<String> outputKeys;
+
+ public ProjectionParameters(DataProcessorInvocation graph, List<String> outputKeys) {
+ super(graph);
+ this.outputKeys = outputKeys;
+ }
+
+ public List<String> getOutputKeys() {
+ return outputKeys;
+ }
+}
diff --git a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/StringOperator.java b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/StringOperator.java
index 6e1cd4b..a59de4e 100644
--- a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/StringOperator.java
+++ b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/StringOperator.java
@@ -17,5 +17,5 @@
package org.streampipes.processors.filters.jvm.processor.textfilter;
public enum StringOperator {
- CONTAINS, MATCHES;
+ CONTAINS, MATCHES
}
diff --git a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/TextFilterController.java b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/TextFilterController.java
index fe26c72..dcf8b83 100644
--- a/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/TextFilterController.java
+++ b/streampipes-processors-filters-jvm/src/main/java/org/streampipes/processors/filters/jvm/processor/textfilter/TextFilterController.java
@@ -41,7 +41,7 @@
@Override
public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("textfilter", "Text Filter", "Text Filter Description")
+ return ProcessingElementBuilder.create("org.streampipes.processors.filters.jvm.textfilter", "Text Filter", "Text Filter Description")
.iconUrl(FiltersJvmConfig.getIconUrl("Textual_Filter_Icon_HQ"))
.category(DataProcessorType.FILTER)
.requiredStream(StreamRequirementsBuilder
@@ -49,8 +49,8 @@
.requiredPropertyWithUnaryMapping(EpRequirements
.stringReq(), Labels.from(MAPPING_PROPERTY_ID, "Select Text Property", ""), PropertyScope.NONE)
.build())
- .requiredSingleValueSelection(OPERATION_ID, "Select Operation", "", Options.from("MATCHES", "CONTAINS"))
- .requiredTextParameter(KEYWORD_ID, "Select Keyword", "", "text")
+ .requiredSingleValueSelection(Labels.from(OPERATION_ID, "Select Operation", ""), Options.from("MATCHES", "CONTAINS"))
+ .requiredTextParameter(Labels.from(KEYWORD_ID, "Select Keyword", ""), "text")
.outputStrategy(OutputStrategies.keep())
.supportedFormats(SupportedFormats.jsonFormat())
.supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
diff --git a/streampipes-processors-filters-siddhi/Dockerfile b/streampipes-processors-filters-siddhi/Dockerfile
new file mode 100644
index 0000000..b326072
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/Dockerfile
@@ -0,0 +1,8 @@
+FROM anapsix/alpine-java
+
+EXPOSE 8090
+ENV CONSUL_LOCATION consul
+
+ADD ./target/streampipes-processors-filters-siddhi.jar /streampipes-processing-element-container.jar
+
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-filters-siddhi/deployment/docker-compose.yml b/streampipes-processors-filters-siddhi/deployment/docker-compose.yml
new file mode 100644
index 0000000..7f20b12
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-filters-siddhi:
+ image: ${SP_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-filters-siddhi:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-filters-siddhi/development/.env b/streampipes-processors-filters-siddhi/development/.env
new file mode 100644
index 0000000..7286d9c
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/development/.env
@@ -0,0 +1,10 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6020
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_COUCHDB_HOST=localhost
+SP_JMS_HOST=localhost
+SP_NGINX_HOST=localhost
+SP_NGINX_PORT=8082
\ No newline at end of file
diff --git a/streampipes-processors-filters-siddhi/pom.xml b/streampipes-processors-filters-siddhi/pom.xml
new file mode 100644
index 0000000..989eb9a
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/pom.xml
@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2018 FZI Forschungszentrum Informatik
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-processors-filters-siddhi</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-siddhi</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-messaging-kafka</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-dataformat-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.streampipes.processors.siddhi.FiltersSiddhiInit</mainClass>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
+ </exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>streampipes-processors-filters-siddhi</finalName>
+
+ </build>
+
+
+</project>
\ No newline at end of file
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/FiltersSiddhiInit.java b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/FiltersSiddhiInit.java
new file mode 100644
index 0000000..82f6477
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/FiltersSiddhiInit.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2017 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.streampipes.processors.siddhi;
+
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.dataformat.json.JsonDataFormatFactory;
+import org.streampipes.messaging.jms.SpJmsProtocolFactory;
+import org.streampipes.messaging.kafka.SpKafkaProtocolFactory;
+import org.streampipes.processors.siddhi.config.FilterSiddhiConfig;
+import org.streampipes.processors.siddhi.filter.NumericalFilterController;
+
+public class FiltersSiddhiInit extends StandaloneModelSubmitter {
+
+ public static void main(String[] args) {
+ DeclarersSingleton
+ .getInstance()
+ .add(new NumericalFilterController());
+
+ DeclarersSingleton.getInstance().registerDataFormat(new JsonDataFormatFactory());
+ DeclarersSingleton.getInstance().registerProtocol(new SpKafkaProtocolFactory());
+ DeclarersSingleton.getInstance().registerProtocol(new SpJmsProtocolFactory());
+
+ new FiltersSiddhiInit().init(FilterSiddhiConfig.INSTANCE);
+ }
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/config/ConfigKeys.java b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/config/ConfigKeys.java
new file mode 100644
index 0000000..0ad5549
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/config/ConfigKeys.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.siddhi.config;
+
+public class ConfigKeys {
+ final static String HOST = "SP_HOST";
+ final static String PORT = "SP_PORT";
+ final static String ICON_HOST = "SP_ICON_HOST";
+ final static String ICON_PORT = "SP_ICON_PORT";
+ final static String KAFKA_HOST = "SP_KAFKA_HOST";
+ final static String KAFKA_PORT = "SP_KAFKA_PORT";
+ final static String ZOOKEEPER_HOST = "SP_ZOOKEEPER_HOST";
+ final static String ZOOKEEPER_PORT = "SP_ZOOKEEPER_PORT";
+ final static String COUCHDB_HOST = "SP_COUCHDB_HOST";
+ final static String COUCHDB_PORT = "SP_COCHDB_PORT";
+ final static String JMS_HOST = "SP_JMS_HOST";
+ final static String JMS_PORT = "SP_JMS_PORT";
+ final static String NGINX_HOST = "SP_NGINX_HOST";
+ final static String NGINX_PORT = "SP_NGINX_PORT";
+ final static String SERVICE_NAME_KEY = "SP_SERVICE_NAME";
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/config/FilterSiddhiConfig.java b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/config/FilterSiddhiConfig.java
new file mode 100644
index 0000000..c168fed
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/config/FilterSiddhiConfig.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2017 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.streampipes.processors.siddhi.config;
+
+
+import org.streampipes.config.SpConfig;
+import org.streampipes.container.model.PeConfig;
+
+public enum FilterSiddhiConfig implements PeConfig {
+ INSTANCE;
+
+ private SpConfig config;
+
+ public final static String serverUrl;
+ public final static String iconBaseUrl;
+
+ private final static String service_id = "pe/org.streampipes.processors.filters.siddhi";
+ private final static String service_name = "Processors Filters Siddhi";
+ private final static String service_container_name = "processors-filters-siddhi";
+
+ FilterSiddhiConfig() {
+ config = SpConfig.getSpConfig(service_id);
+ config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe esper");
+ config.register(ConfigKeys.PORT, 8090, "Port for the pe esper");
+
+ config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host");
+ config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx");
+ config.register(ConfigKeys.NGINX_HOST, "localhost", "External hostname of StreamPipes Nginx");
+ config.register(ConfigKeys.NGINX_PORT, 80, "External port of StreamPipes Nginx");
+ config.register(ConfigKeys.KAFKA_HOST, "kafka", "Host for kafka of the pe sinks project");
+ config.register(ConfigKeys.KAFKA_PORT, 9092, "Port for kafka of the pe sinks project");
+ config.register(ConfigKeys.ZOOKEEPER_HOST, "zookeeper", "Host for zookeeper of the pe sinks project");
+ config.register(ConfigKeys.ZOOKEEPER_PORT, 2181, "Port for zookeeper of the pe sinks project");
+ config.register(ConfigKeys.COUCHDB_HOST, "couchdb", "Host for couchdb of the pe sinks project");
+ config.register(ConfigKeys.COUCHDB_PORT, 5984, "Port for couchdb of the pe sinks project");
+ config.register(ConfigKeys.JMS_HOST, "tcp://activemq", "Hostname for pe actions service for active mq");
+ config.register(ConfigKeys.JMS_PORT, 61616, "Port for pe actions service for active mq");
+
+ config.register(ConfigKeys.SERVICE_NAME_KEY, service_name, "The name of the service");
+
+ }
+
+ static {
+ serverUrl = FilterSiddhiConfig.INSTANCE.getHost() + ":" + FilterSiddhiConfig.INSTANCE.getPort();
+ iconBaseUrl = "http://" + FilterSiddhiConfig.INSTANCE.getIconHost() + ":" + FilterSiddhiConfig.INSTANCE.getIconPort() +"/assets/img/pe_icons";
+ }
+
+ public static final String getIconUrl(String pictureName) {
+ return iconBaseUrl +"/" +pictureName +".png";
+ }
+
+ @Override
+ public String getHost() {
+ return config.getString(ConfigKeys.HOST);
+ }
+
+ @Override
+ public int getPort() {
+ return config.getInteger(ConfigKeys.PORT);
+ }
+
+ public String getIconHost() {
+ return config.getString(ConfigKeys.ICON_HOST);
+ }
+
+ public int getIconPort() {
+ return config.getInteger(ConfigKeys.ICON_PORT);
+ }
+
+ public String getKafkaHost() {
+ return config.getString(ConfigKeys.KAFKA_HOST);
+ }
+
+ public int getKafkaPort() {
+ return config.getInteger(ConfigKeys.KAFKA_PORT);
+ }
+
+ public String getKafkaUrl() {
+ return getKafkaHost() + ":" + getKafkaPort();
+ }
+
+ public String getZookeeperHost() {
+ return config.getString(ConfigKeys.ZOOKEEPER_HOST);
+ }
+
+ public int getZookeeperPort() {
+ return config.getInteger(ConfigKeys.ZOOKEEPER_PORT);
+ }
+
+ public String getCouchDbHost() {
+ return config.getString(ConfigKeys.COUCHDB_HOST);
+ }
+
+ public int getCouchDbPort() {
+ return config.getInteger(ConfigKeys.COUCHDB_PORT);
+ }
+
+ public String getJmsHost() {
+ return config.getString(ConfigKeys.JMS_HOST);
+ }
+
+ public int getJmsPort() {
+ return config.getInteger(ConfigKeys.JMS_PORT);
+ }
+
+ public String getJmsUrl() {
+ return getJmsHost() + ":" + getJmsPort();
+ }
+
+ public String getNginxHost() {
+ return config.getString(ConfigKeys.NGINX_HOST);
+ }
+
+ public Integer getNginxPort() {
+
+ return config.getInteger(ConfigKeys.NGINX_PORT);
+ }
+
+ @Override
+ public String getId() {
+ return service_id;
+ }
+
+ @Override
+ public String getName() {
+ return config.getString(ConfigKeys.SERVICE_NAME_KEY);
+ }
+
+
+
+
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilter.java b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilter.java
new file mode 100644
index 0000000..a75a549
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilter.java
@@ -0,0 +1,38 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.siddhi.filter;
+
+import org.streampipes.wrapper.siddhi.engine.SiddhiEventEngine;
+
+import java.util.List;
+
+public class NumericalFilter extends SiddhiEventEngine<NumericalFilterParameters> {
+
+ public NumericalFilter(NumericalFilterParameters params) {
+ super(params);
+ }
+
+ @Override
+ protected String fromStatement(List<String> inputStreamNames, NumericalFilterParameters params) {
+ return "from " +inputStreamNames.get(0) +"[" +params.getFilterProperty() +"<" +params.getThreshold() +"]";
+ }
+
+ @Override
+ protected String selectStatement(NumericalFilterParameters bindingParameters) {
+ return "select *";
+ }
+
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilterController.java b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilterController.java
new file mode 100644
index 0000000..eed20cd
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilterController.java
@@ -0,0 +1,81 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.siddhi.filter;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+public class NumericalFilterController extends StandaloneEventProcessingDeclarer<NumericalFilterParameters> {
+
+ private static final String NUMBER_MAPPING = "number-mapping";
+ private static final String VALUE = "value";
+ private static final String OPERATION = "operation";
+
+ @Override
+ public ConfiguredEventProcessor<NumericalFilterParameters> onInvocation(DataProcessorInvocation graph) {
+ ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(graph);
+
+ Double threshold = extractor.singleValueParameter(VALUE, Double.class);
+ String stringOperation = extractor.selectedSingleValue(OPERATION, String.class);
+
+ String operation = "GT";
+
+ if (stringOperation.equals("<=")) {
+ operation = "LT";
+ } else if (stringOperation.equals("<")) {
+ operation = "LE";
+ } else if (stringOperation.equals(">=")) {
+ operation = "GE";
+ } else if (stringOperation.equals("==")) {
+ operation = "EQ";
+ }
+
+ String filterProperty = extractor.mappingPropertyValue(NUMBER_MAPPING);
+
+ NumericalFilterParameters staticParam = new NumericalFilterParameters(graph, threshold, NumericalOperator.valueOf
+ (operation)
+ , filterProperty);
+
+ return new ConfiguredEventProcessor<>(staticParam, () -> new NumericalFilter(staticParam));
+ }
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.siddhi.numericalfilter", "Numerical Filter", "Numerical Filter Description")
+ .category(DataProcessorType.FILTER)
+ .iconUrl("Numerical_Filter_Icon_HQ")
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.numberReq(), Labels.from(NUMBER_MAPPING, "Specifies the field name where the filter operation should" +
+ " be applied " +
+ "on.", ""), PropertyScope.NONE).build())
+ .outputStrategy(OutputStrategies.keep())
+ .requiredSingleValueSelection(Labels.from(OPERATION, "Filter Operation", "Specifies the filter " +
+ "operation that should be applied on the field"), Options.from("<", "<=", ">", ">=", "=="))
+ .requiredFloatParameter(Labels.from(VALUE, "Threshold value", "Specifies a threshold value."), NUMBER_MAPPING)
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilterParameters.java b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilterParameters.java
new file mode 100644
index 0000000..38257c3
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalFilterParameters.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2017 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.streampipes.processors.siddhi.filter;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class NumericalFilterParameters extends EventProcessorBindingParams {
+
+ private double threshold;
+ private NumericalOperator numericalOperator;
+ private String filterProperty;
+
+ public NumericalFilterParameters(DataProcessorInvocation graph, Double threshold, NumericalOperator
+ numericalOperator, String filterProperty) {
+ super(graph);
+ this.threshold = threshold;
+ this.numericalOperator = numericalOperator;
+ this.filterProperty = filterProperty;
+ }
+
+ public double getThreshold() {
+ return threshold;
+ }
+
+ public NumericalOperator getNumericalOperator() {
+ return numericalOperator;
+ }
+
+ public String getFilterProperty() {
+ return filterProperty;
+ }
+}
diff --git a/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalOperator.java b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalOperator.java
new file mode 100644
index 0000000..c1a3d5c
--- /dev/null
+++ b/streampipes-processors-filters-siddhi/src/main/java/org/streampipes/processors/siddhi/filter/NumericalOperator.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2017 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.streampipes.processors.siddhi.filter;
+
+public enum NumericalOperator {
+GE, GT, LE, LT, EQ
+}
diff --git a/streampipes-processors-geo-flink/Dockerfile b/streampipes-processors-geo-flink/Dockerfile
new file mode 100644
index 0000000..d3d235a
--- /dev/null
+++ b/streampipes-processors-geo-flink/Dockerfile
@@ -0,0 +1,8 @@
+FROM anapsix/alpine-java
+
+EXPOSE 8090
+ENV CONSUL_LOCATION consul
+
+ADD ./target/streampipes-processors-geo-flink.jar /streampipes-processing-element-container.jar
+
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-geo-flink/deployment/docker-compose.yml b/streampipes-processors-geo-flink/deployment/docker-compose.yml
new file mode 100644
index 0000000..8b54749
--- /dev/null
+++ b/streampipes-processors-geo-flink/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-geo-flink:
+ image: ${SP_PE_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-geo-flink:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-geo-flink/development/.env b/streampipes-processors-geo-flink/development/.env
new file mode 100644
index 0000000..9812e16
--- /dev/null
+++ b/streampipes-processors-geo-flink/development/.env
@@ -0,0 +1,5 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6025
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_FLINK_DEBUG=true
diff --git a/streampipes-processors-geo-flink/pom.xml b/streampipes-processors-geo-flink/pom.xml
new file mode 100644
index 0000000..a775e77
--- /dev/null
+++ b/streampipes-processors-geo-flink/pom.xml
@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2018 FZI Forschungszentrum Informatik
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-processors-geo-flink</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-commons</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-flink</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.9.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.streampipes.processor.geo.flink.GeoFlinkInit</mainClass>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
+ </exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>streampipes-processors-geo-flink</finalName>
+ </build>
+</project>
\ No newline at end of file
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/AbstractGeoProgram.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/AbstractGeoProgram.java
new file mode 100644
index 0000000..3b62b34
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/AbstractGeoProgram.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processor.geo.flink;
+
+import org.streampipes.processor.geo.flink.config.GeoFlinkConfig;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public abstract class AbstractGeoProgram<B extends EventProcessorBindingParams> extends FlinkDataProcessorRuntime<B> {
+
+ public AbstractGeoProgram(B params, boolean debug) {
+ super(params, debug);
+ }
+
+ public AbstractGeoProgram(B params) {
+ super(params, false);
+ }
+
+ @Override
+ protected FlinkDeploymentConfig getDeploymentConfig() {
+ return new FlinkDeploymentConfig(GeoFlinkConfig.JAR_FILE,
+ GeoFlinkConfig.INSTANCE.getFlinkHost(), GeoFlinkConfig.INSTANCE.getFlinkPort());
+ }
+
+}
+
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/GeoFlinkInit.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/GeoFlinkInit.java
new file mode 100644
index 0000000..29592c9
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/GeoFlinkInit.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink;
+
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.processor.geo.flink.config.GeoFlinkConfig;
+import org.streampipes.processor.geo.flink.processor.gridenricher.SpatialGridEnrichmentController;
+
+public class GeoFlinkInit extends StandaloneModelSubmitter {
+
+ public static void main(String[] args) {
+ DeclarersSingleton.getInstance()
+ .add(new SpatialGridEnrichmentController());
+
+ new GeoFlinkInit().init(GeoFlinkConfig.INSTANCE);
+ }
+
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/config/ConfigKeys.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/config/ConfigKeys.java
new file mode 100644
index 0000000..1c58510
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/config/ConfigKeys.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.config;
+
+public class ConfigKeys {
+ final static String HOST = "SP_HOST";
+ final static String PORT = "SP_PORT";
+ final static String FLINK_HOST = "SP_FLINK_HOST";
+ final static String FLINK_PORT = "SP_FLINK_PORT";
+ final static String ICON_HOST = "SP_ICON_HOST";
+ final static String ICON_PORT = "SP_ICON_PORT";
+ final static String SERVICE_NAME = "SP_SERVICE_NAME";
+ final static String DEBUG = "SP_FLINK_DEBUG";
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/config/GeoFlinkConfig.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/config/GeoFlinkConfig.java
new file mode 100644
index 0000000..a0c7adc
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/config/GeoFlinkConfig.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.config;
+
+import org.streampipes.config.SpConfig;
+import org.streampipes.container.model.PeConfig;
+
+public enum GeoFlinkConfig implements PeConfig {
+ INSTANCE;
+
+ private SpConfig config;
+ public static final String JAR_FILE = "./streampipes-processing-element-container.jar";
+
+ private final static String service_id = "pe/org.streampipes.processors.geo.flink";
+ private final static String service_name = "Processors Geo Flink";
+ private final static String service_container_name = "processors-geo-flink";
+ GeoFlinkConfig() {
+ config = SpConfig.getSpConfig(service_id);
+
+ config.register(ConfigKeys.HOST, service_container_name, "Hostname for the geo flink component");
+ config.register(ConfigKeys.PORT, 8090, "Port for the geo flink component");
+ config.register(ConfigKeys.FLINK_HOST, "jobmanager", "Host for the flink cluster");
+ config.register(ConfigKeys.FLINK_PORT, 6123, "Port for the flink cluster");
+ config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host");
+ config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx");
+
+ config.register(ConfigKeys.DEBUG, false, "When set to true programs are not deployed to cluster, but executed locally");
+
+ config.register(ConfigKeys.SERVICE_NAME, service_name, "The name of the service");
+
+ }
+
+ @Override
+ public String getHost() {
+ return config.getString(ConfigKeys.HOST);
+ }
+
+ @Override
+ public int getPort() {
+ return config.getInteger(ConfigKeys.PORT);
+ }
+
+ public String getFlinkHost() {
+ return config.getString(ConfigKeys.FLINK_HOST);
+ }
+
+ public int getFlinkPort() {
+ return config.getInteger(ConfigKeys.FLINK_PORT);
+ }
+
+ public static final String iconBaseUrl = "http://" + GeoFlinkConfig.INSTANCE.getIconHost() + ":" + GeoFlinkConfig.INSTANCE.getIconPort() + "/assets/img/pe_icons";
+
+ public static final String getIconUrl(String pictureName) {
+ return iconBaseUrl + "/" + pictureName + ".png";
+ }
+
+ public String getIconHost() {
+ return config.getString(ConfigKeys.ICON_HOST);
+ }
+
+ public int getIconPort() {
+ return config.getInteger(ConfigKeys.ICON_PORT);
+ }
+
+ public boolean getDebug() {
+ return config.getBoolean(ConfigKeys.DEBUG);
+ }
+
+ @Override
+ public String getId() {
+ return service_id;
+ }
+
+ @Override
+ public String getName() {
+ return config.getString(ConfigKeys.SERVICE_NAME);
+ }
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/CellOption.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/CellOption.java
new file mode 100644
index 0000000..4d40981
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/CellOption.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+public class CellOption {
+ private int cellX;
+ private int cellY;
+
+ private double latitudeNW;
+ private double longitudeNW;
+ private double latitudeSE;
+ private double longitudeSE;
+
+ private int cellSize;
+
+ public CellOption(int cellX, int cellY, double latitudeNW,
+ double longitudeNW, double latitudeSE, double longitudeSE,
+ int cellSize) {
+ super();
+ this.cellX = cellX;
+ this.cellY = cellY;
+ this.latitudeNW = latitudeNW;
+ this.longitudeNW = longitudeNW;
+ this.latitudeSE = latitudeSE;
+ this.longitudeSE = longitudeSE;
+ this.cellSize = cellSize;
+ }
+
+
+ public int getCellX() {
+ return cellX;
+ }
+
+
+ public void setCellX(int cellX) {
+ this.cellX = cellX;
+ }
+
+
+ public int getCellY() {
+ return cellY;
+ }
+
+
+ public void setCellY(int cellY) {
+ this.cellY = cellY;
+ }
+
+
+ public double getLatitudeNW() {
+ return latitudeNW;
+ }
+
+ public void setLatitudeNW(double latitudeNW) {
+ this.latitudeNW = latitudeNW;
+ }
+
+ public double getLongitudeNW() {
+ return longitudeNW;
+ }
+
+ public void setLongitudeNW(double longitudeNW) {
+ this.longitudeNW = longitudeNW;
+ }
+
+ public double getLatitudeSE() {
+ return latitudeSE;
+ }
+
+ public void setLatitudeSE(double latitudeSE) {
+ this.latitudeSE = latitudeSE;
+ }
+
+ public double getLongitudeSE() {
+ return longitudeSE;
+ }
+
+ public void setLongitudeSE(double longitudeSE) {
+ this.longitudeSE = longitudeSE;
+ }
+
+ public int getCellSize() {
+ return cellSize;
+ }
+
+ public void setCellSize(int cellSize) {
+ this.cellSize = cellSize;
+ }
+
+ @Override
+ public boolean equals(Object other)
+ {
+ if (other == this) return true;
+ if (!(other instanceof CellOption)) return false;
+ else
+ {
+ CellOption c2 = (CellOption) other;
+ return c2.getCellSize() == cellSize
+ && c2.getCellX() == cellX
+ && c2.getCellY() == cellY
+ && c2.getLatitudeNW() == latitudeNW
+ && c2.getLatitudeSE() == latitudeSE
+ && c2.getLongitudeNW() == longitudeNW
+ && c2.getLongitudeSE() == longitudeSE;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 1;
+ hash = hash * 17 + cellSize;
+ hash = hash * 31 + cellX;
+ hash = hash * 13 + cellY;
+ return hash;
+ }
+}
\ No newline at end of file
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/EnrichmentSettings.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/EnrichmentSettings.java
new file mode 100644
index 0000000..47d2bc3
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/EnrichmentSettings.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+import java.io.Serializable;
+
+public class EnrichmentSettings implements Serializable {
+
+ private double latitudeStart;
+ private double longitudeStart;
+
+ private int cellSize;
+
+ private String latPropertyName;
+ private String lngPropertyName;
+
+ public EnrichmentSettings() {
+ }
+
+ public EnrichmentSettings(double latitudeStart, double longitudeStart, int cellSize, String latPropertyName, String lngPropertyName) {
+ this.latitudeStart = latitudeStart;
+ this.longitudeStart = longitudeStart;
+ this.cellSize = cellSize;
+ this.latPropertyName = latPropertyName;
+ this.lngPropertyName = lngPropertyName;
+ }
+
+ public double getLatitudeStart() {
+ return latitudeStart;
+ }
+
+ public void setLatitudeStart(double latitudeStart) {
+ this.latitudeStart = latitudeStart;
+ }
+
+ public double getLongitudeStart() {
+ return longitudeStart;
+ }
+
+ public void setLongitudeStart(double longitudeStart) {
+ this.longitudeStart = longitudeStart;
+ }
+
+ public int getCellSize() {
+ return cellSize;
+ }
+
+ public void setCellSize(int cellSize) {
+ this.cellSize = cellSize;
+ }
+
+ public String getLatPropertyName() {
+ return latPropertyName;
+ }
+
+ public void setLatPropertyName(String latPropertyName) {
+ this.latPropertyName = latPropertyName;
+ }
+
+ public String getLngPropertyName() {
+ return lngPropertyName;
+ }
+
+ public void setLngPropertyName(String lngPropertyName) {
+ this.lngPropertyName = lngPropertyName;
+ }
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridCalculator.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridCalculator.java
new file mode 100644
index 0000000..f749dc4
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridCalculator.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+import java.io.Serializable;
+
+public class SpatialGridCalculator implements Serializable {
+
+ private double SOUTHDIFF = 0.004491556;
+ private double EASTDIFF = 0.005986;
+
+ private Integer cellSize;
+ private Double startLat;
+ private Double startLon;
+
+ private EnrichmentSettings settings;
+
+ public SpatialGridCalculator(EnrichmentSettings settings) {
+ this.settings = settings;
+ this.startLat = settings.getLatitudeStart();
+ this.startLon = settings.getLongitudeStart();
+ this.cellSize = settings.getCellSize();
+ this.SOUTHDIFF = (cellSize / 500.0) * SOUTHDIFF;
+ this.EASTDIFF = (cellSize / 500.0) * EASTDIFF;
+ }
+
+ public CellOption computeCells(double latitude, double longitude) {
+
+ int cellX = calculateXCoordinate(longitude);
+ int cellY = calculateYCoordinate(latitude);
+
+ return new CellOption(cellX, cellY, startLat - (cellY * SOUTHDIFF),
+ startLon + (cellX *
+ EASTDIFF), startLat - ((cellY + 1) * SOUTHDIFF), startLon + (
+ (cellX + 1) * EASTDIFF),
+ cellSize);
+ }
+
+ private int calculateXCoordinate(double longitude) {
+ Double ret = (Math.abs(startLon - longitude) / EASTDIFF);
+ return ret.intValue() + 1;
+ }
+
+ private int calculateYCoordinate(double latitude) {
+ Double ret = (Math.abs(startLat - latitude) / SOUTHDIFF);
+ return ret.intValue() + 1;
+ }
+
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridConstants.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridConstants.java
new file mode 100644
index 0000000..c66a8c8
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridConstants.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+public class SpatialGridConstants {
+
+ public static final String GRID_X_KEY = "grid-cell-x";
+ public static final String GRID_Y_KEY = "grid-cell-y";
+ public static final String GRID_LAT_NW_KEY = "grid-latitude-nw";
+ public static final String GRID_LON_NW_KEY = "grid-longitude-nw";
+ public static final String GRID_LAT_SE_KEY = "grid-latitude-se";
+ public static final String GRID_LON_SE_KEY = "grid-longitude-se";
+ public static final String GRID_CELLSIZE_KEY = "grid-cellsize";
+
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnricher.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnricher.java
new file mode 100644
index 0000000..20a9e5d
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnricher.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+
+import java.util.Map;
+
+public class SpatialGridEnricher implements FlatMapFunction<Map<String, Object>, Map<String, Object>> {
+
+ private EnrichmentSettings settings;
+ private SpatialGridCalculator calculator;
+
+ public SpatialGridEnricher(EnrichmentSettings settings) {
+ this.settings = settings;
+ this.calculator = new SpatialGridCalculator(settings);
+ }
+
+ @Override
+ public void flatMap(Map<String, Object> in, Collector<Map<String, Object>> out) throws
+ Exception {
+ Double latitude = toDouble(in.get(settings.getLatPropertyName()));
+ Double longitude = toDouble(in.get(settings.getLngPropertyName()));
+
+ CellOption result = calculator.computeCells(latitude, longitude);
+// System.out.println("x=" +result.getCellX() +", y=" +result.getCellY());
+
+ out.collect(toOutput(in, result));
+ }
+
+ private Map<String,Object> toOutput(Map<String, Object> in, CellOption result) {
+ in.put(SpatialGridConstants.GRID_X_KEY, result.getCellX());
+ in.put(SpatialGridConstants.GRID_Y_KEY, result.getCellY());
+ in.put(SpatialGridConstants.GRID_CELLSIZE_KEY, result.getCellSize());
+ in.put(SpatialGridConstants.GRID_LAT_NW_KEY, result.getLatitudeNW());
+ in.put(SpatialGridConstants.GRID_LON_NW_KEY, result.getLongitudeNW());
+ in.put(SpatialGridConstants.GRID_LAT_SE_KEY, result.getLatitudeSE());
+ in.put(SpatialGridConstants.GRID_LON_SE_KEY, result.getLongitudeSE());
+
+ return in;
+ }
+
+ private Double toDouble(Object value) {
+ return Double.parseDouble(String.valueOf(value));
+ }
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentController.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentController.java
new file mode 100644
index 0000000..93d6887
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentController.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processor.geo.flink.config.GeoFlinkConfig;
+import org.streampipes.sdk.StaticProperties;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.vocabulary.Geo;
+import org.streampipes.vocabulary.SO;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class SpatialGridEnrichmentController extends FlinkDataProcessorDeclarer<SpatialGridEnrichmentParameters> {
+
+ private static final String MAPPING_LATITUDE = "mapping-latitude";
+ private static final String MAPPING_LONGITUDE = "mapping-longitude";
+
+ private static final String CELLSIZE = "cellsize";
+ private static final String STARTING_CELL = "starting-cell";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processor.geo.flink", "Spatial Grid Enrichment", "Groups spatial " +
+ "events into cells of a given size")
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq(Geo.lat)
+ , Labels.from(MAPPING_LATITUDE, "Latitude Property", ""), PropertyScope.MEASUREMENT_PROPERTY)
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq(Geo.lng)
+ , Labels.from(MAPPING_LONGITUDE, "Longitude Property", ""), PropertyScope.MEASUREMENT_PROPERTY)
+ .build())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .outputStrategy(OutputStrategies.append(
+ EpProperties.integerEp(Labels.empty(), SpatialGridConstants.GRID_X_KEY, SO.Number),
+ EpProperties.integerEp(Labels.empty(), SpatialGridConstants.GRID_Y_KEY, SO.Number),
+ EpProperties.doubleEp(Labels.empty(), SpatialGridConstants.GRID_LAT_NW_KEY, Geo.lat),
+ EpProperties.doubleEp(Labels.empty(), SpatialGridConstants.GRID_LON_NW_KEY, Geo.lng),
+ EpProperties.doubleEp(Labels.empty(), SpatialGridConstants.GRID_LAT_SE_KEY, Geo.lat),
+ EpProperties.doubleEp(Labels.empty(), SpatialGridConstants.GRID_LON_SE_KEY, Geo.lng),
+ EpProperties.integerEp(Labels.empty(), SpatialGridConstants.GRID_CELLSIZE_KEY, SO.Number)))
+ .requiredIntegerParameter(CELLSIZE, "Cell Size", "The size of a cell in meters",
+ 100, 10000, 100)
+ .requiredOntologyConcept(Labels.from(STARTING_CELL, "Starting Location", "The " +
+ "upper-left corner of the starting cell"), StaticProperties
+ .supportedDomainProperty(Geo.lat, true), StaticProperties
+ .supportedDomainProperty(Geo.lng, true))
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<SpatialGridEnrichmentParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+
+ Integer cellSize = extractor.singleValueParameter(CELLSIZE, Integer.class);
+ String latitudePropertyName = extractor.mappingPropertyValue(MAPPING_LATITUDE);
+ String longitudePropertyName = extractor.mappingPropertyValue(MAPPING_LONGITUDE);
+
+ Double startingLatitude = extractor.supportedOntologyPropertyValue(STARTING_CELL, Geo.lat,
+ Double.class);
+
+ Double startingLongitude = extractor.supportedOntologyPropertyValue(STARTING_CELL, Geo.lng,
+ Double.class);
+
+ EnrichmentSettings enrichmentSettings = new EnrichmentSettings(
+ startingLatitude, startingLongitude,
+ cellSize,
+ latitudePropertyName,
+ longitudePropertyName);
+
+ SpatialGridEnrichmentParameters params = new SpatialGridEnrichmentParameters(graph,
+ enrichmentSettings);
+
+ return new SpatialGridEnrichmentProgram(params, GeoFlinkConfig.INSTANCE.getDebug());
+
+ }
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentParameters.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentParameters.java
new file mode 100644
index 0000000..d8432f5
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentParameters.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class SpatialGridEnrichmentParameters extends EventProcessorBindingParams {
+
+ private EnrichmentSettings enrichmentSettings;
+
+ public SpatialGridEnrichmentParameters(DataProcessorInvocation graph, EnrichmentSettings enrichmentSettings) {
+ super(graph);
+ this.enrichmentSettings = enrichmentSettings;
+ }
+
+ public EnrichmentSettings getEnrichmentSettings() {
+ return enrichmentSettings;
+ }
+}
diff --git a/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentProgram.java b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentProgram.java
new file mode 100644
index 0000000..0090319
--- /dev/null
+++ b/streampipes-processors-geo-flink/src/main/java/org/streampipes/processor/geo/flink/processor/gridenricher/SpatialGridEnrichmentProgram.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processor.geo.flink.processor.gridenricher;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processor.geo.flink.AbstractGeoProgram;
+
+import java.util.Map;
+
+public class SpatialGridEnrichmentProgram extends AbstractGeoProgram<SpatialGridEnrichmentParameters> {
+
+ public SpatialGridEnrichmentProgram(SpatialGridEnrichmentParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>[] messageStream) {
+ return messageStream[0].flatMap(new SpatialGridEnricher(params.getEnrichmentSettings()));
+ }
+}
diff --git a/streampipes-processors-geo-jvm/Dockerfile b/streampipes-processors-geo-jvm/Dockerfile
new file mode 100644
index 0000000..8c8cc67
--- /dev/null
+++ b/streampipes-processors-geo-jvm/Dockerfile
@@ -0,0 +1,8 @@
+FROM anapsix/alpine-java
+
+EXPOSE 8090
+ENV CONSUL_LOCATION consul
+
+ADD ./target/streampipes-processors-geo-jvm.jar /streampipes-processing-element-container.jar
+
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-geo-jvm/deployment/docker-compose.yml b/streampipes-processors-geo-jvm/deployment/docker-compose.yml
new file mode 100644
index 0000000..1f931ae
--- /dev/null
+++ b/streampipes-processors-geo-jvm/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-geo-jvm:
+ image: ${SP_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-geo-jvm:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-geo-jvm/development/.env b/streampipes-processors-geo-jvm/development/.env
new file mode 100644
index 0000000..d458efa
--- /dev/null
+++ b/streampipes-processors-geo-jvm/development/.env
@@ -0,0 +1,8 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6030
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_NGINX_HOST=localhost
+SP_NGINX_PORT=8082
\ No newline at end of file
diff --git a/streampipes-processors-geo-jvm/pom.xml b/streampipes-processors-geo-jvm/pom.xml
new file mode 100644
index 0000000..d32ba6f
--- /dev/null
+++ b/streampipes-processors-geo-jvm/pom.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-processors-geo-jvm</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-standalone</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-dataformat-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-messaging-kafka</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.squareup.okhttp3</groupId>
+ <artifactId>okhttp</artifactId>
+ <version>3.9.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.maps</groupId>
+ <artifactId>google-maps-services</artifactId>
+ <version>0.2.6</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.streampipes.processors.geo.jvm.GeoJvmInit</mainClass>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
+ </exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>streampipes-processors-geo-jvm</finalName>
+
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/GeoJvmInit.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/GeoJvmInit.java
new file mode 100644
index 0000000..e7c8d3d
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/GeoJvmInit.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.geo.jvm;
+
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.dataformat.json.JsonDataFormatFactory;
+import org.streampipes.messaging.jms.SpJmsProtocolFactory;
+import org.streampipes.messaging.kafka.SpKafkaProtocolFactory;
+import org.streampipes.processors.geo.jvm.config.GeoJvmConfig;
+import org.streampipes.processors.geo.jvm.processor.geocode.GeocodingController;
+import org.streampipes.processors.geo.jvm.processor.route.GoogleRoutingController;
+
+public class GeoJvmInit extends StandaloneModelSubmitter {
+
+ public static void main(String[] args) {
+ DeclarersSingleton
+ .getInstance()
+ .add(new GeocodingController())
+ .add(new GoogleRoutingController());
+
+ DeclarersSingleton.getInstance().registerDataFormat(new JsonDataFormatFactory());
+ DeclarersSingleton.getInstance().registerProtocol(new SpKafkaProtocolFactory());
+ DeclarersSingleton.getInstance().registerProtocol(new SpJmsProtocolFactory());
+
+ new GeoJvmInit().init(GeoJvmConfig.INSTANCE);
+ }
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/config/ConfigKeys.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/config/ConfigKeys.java
new file mode 100644
index 0000000..44814fd
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/config/ConfigKeys.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.geo.jvm.config;
+
+public class ConfigKeys {
+ final static String HOST = "SP_HOST";
+ final static String PORT = "SP_PORT";
+ final static String ICON_HOST = "SP_ICON_HOST";
+ final static String ICON_PORT = "SP_ICON_PORT";
+ final static String KAFKA_HOST = "SP_KAFKA_HOST";
+ final static String KAFKA_PORT = "SP_KAFKA_PORT";
+ final static String ZOOKEEPER_HOST = "SP_ZOOKEEPER_HOST";
+ final static String ZOOKEEPER_PORT = "SP_ZOOKEEPER_PORT";
+ final static String NGINX_HOST = "SP_NGINX_HOST";
+ final static String NGINX_PORT = "SP_NGINX_PORT";
+ final static String SERVICE_NAME_KEY = "SP_SERVICE_NAME";
+ final static String GOOGLE_API_KEY = "SP_GOOGLE_API_KEY";
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/config/GeoJvmConfig.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/config/GeoJvmConfig.java
new file mode 100644
index 0000000..46c76f4
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/config/GeoJvmConfig.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.geo.jvm.config;
+
+
+import org.streampipes.config.SpConfig;
+import org.streampipes.container.model.PeConfig;
+
+public enum GeoJvmConfig implements PeConfig {
+ INSTANCE;
+
+ private SpConfig config;
+
+ public final static String serverUrl;
+ public final static String iconBaseUrl;
+
+ private final static String service_id = "pe/org.streampipes.processors.geo.jvm";
+ private final static String service_name = "Processors Geo JVM";
+ private final static String service_container_name = "processors-geo-jvm";
+
+ GeoJvmConfig() {
+ config = SpConfig.getSpConfig(service_id);
+ config.register(ConfigKeys.HOST, service_container_name, "Hostname for the geo container");
+ config.register(ConfigKeys.PORT, 8090, "Port for the pe esper");
+
+ config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host");
+ config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx");
+ config.register(ConfigKeys.NGINX_HOST, "localhost", "External hostname of StreamPipes Nginx");
+ config.register(ConfigKeys.NGINX_PORT, 80, "External port of StreamPipes Nginx");
+ config.register(ConfigKeys.KAFKA_HOST, "kafka", "Host for kafka of the pe sinks project");
+ config.register(ConfigKeys.KAFKA_PORT, 9092, "Port for kafka of the pe sinks project");
+ config.register(ConfigKeys.ZOOKEEPER_HOST, "zookeeper", "Host for zookeeper of the pe sinks project");
+ config.register(ConfigKeys.ZOOKEEPER_PORT, 2181, "Port for zookeeper of the pe sinks project");
+
+ config.registerPassword(ConfigKeys.GOOGLE_API_KEY, "", "Google API Key for the routing service");
+
+ config.register(ConfigKeys.SERVICE_NAME_KEY, service_name, "The name of the service");
+
+ }
+
+ static {
+ serverUrl = GeoJvmConfig.INSTANCE.getHost() + ":" + GeoJvmConfig.INSTANCE.getPort();
+ iconBaseUrl = "http://" + GeoJvmConfig.INSTANCE.getIconHost() + ":" + GeoJvmConfig.INSTANCE.getIconPort() + "/assets/img/pe_icons";
+ }
+
+ public static final String getIconUrl(String pictureName) {
+ return iconBaseUrl + "/" + pictureName + ".png";
+ }
+
+ @Override
+ public String getHost() {
+ return config.getString(ConfigKeys.HOST);
+ }
+
+ @Override
+ public int getPort() {
+ return config.getInteger(ConfigKeys.PORT);
+ }
+
+ public String getIconHost() {
+ return config.getString(ConfigKeys.ICON_HOST);
+ }
+
+ public int getIconPort() {
+ return config.getInteger(ConfigKeys.ICON_PORT);
+ }
+
+ public String getKafkaHost() {
+ return config.getString(ConfigKeys.KAFKA_HOST);
+ }
+
+ public int getKafkaPort() {
+ return config.getInteger(ConfigKeys.KAFKA_PORT);
+ }
+
+ public String getKafkaUrl() {
+ return getKafkaHost() + ":" + getKafkaPort();
+ }
+
+ public String getZookeeperHost() {
+ return config.getString(ConfigKeys.ZOOKEEPER_HOST);
+ }
+
+ public int getZookeeperPort() {
+ return config.getInteger(ConfigKeys.ZOOKEEPER_PORT);
+ }
+
+ public String getNginxHost() {
+ return config.getString(ConfigKeys.NGINX_HOST);
+ }
+
+ public Integer getNginxPort() {
+
+ return config.getInteger(ConfigKeys.NGINX_PORT);
+ }
+
+ public String getGoogleApiKey() {
+ return config.getString(ConfigKeys.GOOGLE_API_KEY);
+ }
+
+ @Override
+ public String getId() {
+ return service_id;
+ }
+
+ @Override
+ public String getName() {
+ return config.getString(ConfigKeys.SERVICE_NAME_KEY);
+ }
+
+
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/Geocoder.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/Geocoder.java
new file mode 100644
index 0000000..249c459
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/Geocoder.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.geo.jvm.processor.geocode;
+
+import com.google.maps.GeoApiContext;
+import com.google.maps.GeocodingApi;
+import com.google.maps.errors.ApiException;
+import com.google.maps.model.GeocodingResult;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.geo.jvm.config.GeoJvmConfig;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class Geocoder extends StandaloneEventProcessorEngine<GeocodingParameters> {
+
+ private GeocodingParameters geocodingParameters;
+ private GeoApiContext context;
+
+ public Geocoder(GeocodingParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(GeocodingParameters geocodingParameters, DataProcessorInvocation dataProcessorInvocation) {
+ this.geocodingParameters = geocodingParameters;
+ context = new GeoApiContext.Builder()
+ .apiKey(GeoJvmConfig.INSTANCE.getGoogleApiKey())
+ .build();
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> in, String s, SpOutputCollector spOutputCollector) {
+ String city = String.valueOf(in.get(geocodingParameters.getCity()));
+ String street = String.valueOf(in.get(geocodingParameters.getStreet()));
+ String number = String.valueOf(in.get(geocodingParameters.getNumber()));
+
+ String searchQuery = street + " " +number + ", " +city;
+
+ try {
+ GeocodingResult[] result = GeocodingApi.geocode(context, searchQuery).await();
+ if(result.length > 0) {
+ in.put("latitude", result[0].geometry.location.lat);
+ in.put("longitude", result[0].geometry.location.lng);
+ }
+ } catch (ApiException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ spOutputCollector.onEvent(in);
+
+
+ }
+
+ @Override
+ public void onDetach() {
+
+ }
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/GeocodingController.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/GeocodingController.java
new file mode 100644
index 0000000..57ceb82
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/GeocodingController.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.geo.jvm.processor.geocode;
+
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.geo.jvm.config.GeoJvmConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.vocabulary.Geo;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+public class GeocodingController extends StandaloneEventProcessingDeclarer<GeocodingParameters> {
+
+ private static final String CITY_MAPPING = "city";
+ private static final String STREET_MAPPING = "street";
+ private static final String STREET_NUMBER_MAPPING = "street-number";
+
+ @Override
+ public ConfiguredEventProcessor<GeocodingParameters> onInvocation(DataProcessorInvocation
+ graph) {
+ ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(graph);
+
+ String city = extractor.mappingPropertyValue(CITY_MAPPING);
+ String street = extractor.mappingPropertyValue(STREET_MAPPING);
+ String number = extractor.mappingPropertyValue(STREET_NUMBER_MAPPING);
+
+ GeocodingParameters params = new GeocodingParameters(graph, city, street, number);
+ return new ConfiguredEventProcessor<>(params, () -> new Geocoder(params));
+ }
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.geo.jvm.geocoder", "Geocoder", "Geocodes a location based " +
+ "on given street, address and street number")
+ .iconUrl(GeoJvmConfig.iconBaseUrl + "Location_Icon_HQ.png")
+
+ .requiredStream(
+ StreamRequirementsBuilder.create()
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements
+ .domainPropertyReq("http://schema.org/city"),
+ Labels.from(CITY_MAPPING, "City", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements
+ .domainPropertyReq("http://schema" +
+ ".org/streetAddress"),
+ Labels.from(STREET_MAPPING, "Street", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements
+ .domainPropertyReq("http://schema" +
+ ".org/streetNumber"),
+ Labels.from(STREET_NUMBER_MAPPING, "Street number", ""),
+ PropertyScope.NONE)
+ .build())
+ .outputStrategy(OutputStrategies.append(EpProperties.doubleEp(Labels.from("latitude",
+ "Latitude", ""),
+ "latitude", Geo.lat), EpProperties.doubleEp(Labels.from("longitude",
+ "Longitude", ""), "longitude", Geo.lng)))
+
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .build();
+ }
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/GeocodingParameters.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/GeocodingParameters.java
new file mode 100644
index 0000000..3e2e615
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/geocode/GeocodingParameters.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.geo.jvm.processor.geocode;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class GeocodingParameters extends EventProcessorBindingParams {
+
+ private String city;
+ private String street;
+ private String number;
+
+ public GeocodingParameters(DataProcessorInvocation graph, String city, String street, String number) {
+ super(graph);
+ this.city = city;
+ this.street = street;
+ this.number = number;
+ }
+
+ public String getCity() {
+ return city;
+ }
+
+ public String getStreet() {
+ return street;
+ }
+
+ public String getNumber() {
+ return number;
+ }
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRouting.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRouting.java
new file mode 100644
index 0000000..84b4610
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRouting.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.geo.jvm.processor.route;
+
+import com.google.maps.DistanceMatrixApi;
+import com.google.maps.GeoApiContext;
+import com.google.maps.errors.ApiException;
+import com.google.maps.model.DistanceMatrix;
+import com.google.maps.model.LatLng;
+import org.streampipes.logging.api.Logger;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.geo.jvm.config.GeoJvmConfig;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+public class GoogleRouting extends StandaloneEventProcessorEngine<GoogleRoutingParameters> {
+
+ private static Logger LOG;
+
+ private GoogleRoutingParameters googleRoutingParameters;
+ private GeoApiContext context;
+
+ public GoogleRouting(GoogleRoutingParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(GoogleRoutingParameters googleRoutingParameters, DataProcessorInvocation dataProcessorInvocation) {
+ LOG = googleRoutingParameters.getGraph().getLogger(GoogleRouting.class);
+
+ this.googleRoutingParameters = googleRoutingParameters;
+ context = new GeoApiContext.Builder()
+ .apiKey(GeoJvmConfig.INSTANCE.getGoogleApiKey())
+ .build();
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> in, String s, SpOutputCollector out) {
+ String city = (String) in.get(googleRoutingParameters.getCity());
+ String street = (String) in.get(googleRoutingParameters.getStreet());
+ String number = (String) in.get(googleRoutingParameters.getNumber());
+ String home = googleRoutingParameters.getHome();
+
+ String destinationLocation = city + ", " + street + ", " + number;
+
+ try {
+
+ String[] origin = {home};
+ String[] destination = {destinationLocation};
+ DistanceMatrix rest = DistanceMatrixApi.getDistanceMatrix(context, origin, destination).await();
+
+ if (rest.rows.length > 0 && rest.rows[0].elements[0].status.name().equals("NOT_FOUND")) {
+ LOG.info("Could not find location: " + destinationLocation);
+ } else {
+
+ long l = rest.rows[0].elements[0].distance.inMeters;
+
+ in.put("kvi", l);
+
+ out.onEvent(in);
+ }
+
+ } catch (ApiException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ private static Map<String, Object> getGeoObject(LatLng latLng) {
+ Map<String, Object> result = new HashMap<>();
+
+ result.put("latitude", latLng.lat);
+ result.put("longitude", latLng.lng);
+
+ return result;
+
+ }
+
+ @Override
+ public void onDetach() {
+ }
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRoutingController.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRoutingController.java
new file mode 100644
index 0000000..8427791
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRoutingController.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.geo.jvm.processor.route;
+
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.geo.jvm.config.GeoJvmConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+public class GoogleRoutingController extends StandaloneEventProcessingDeclarer<GoogleRoutingParameters> {
+
+ private static final String CITY_MAPPING = "city";
+ private static final String STREET_MAPPING = "street";
+ private static final String STREET_NUMBER_MAPPING = "street-number";
+ private static final String START_ADDRESS = "start-address";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.geo.jvm.google-routing", "Google Routing", "Uses the Google" +
+ "routing service to calculate a route from start to destination")
+ .iconUrl(GeoJvmConfig.iconBaseUrl + "Map_Icon_HQ.png")
+ .requiredStream(
+ StreamRequirementsBuilder.create()
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements
+ .domainPropertyReq("http://schema.org/city"),
+ Labels.from(CITY_MAPPING, "City", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements
+ .domainPropertyReq("http://schema" +
+ ".org/streetAddress"),
+ Labels.from(STREET_MAPPING, "Street", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements
+ .domainPropertyReq("http://schema" +
+ ".org/streetNumber"),
+ Labels.from(STREET_NUMBER_MAPPING, "Street Number", ""),
+ PropertyScope.NONE)
+ .build())
+
+ .requiredTextParameter(Labels.from(START_ADDRESS, "Start Address", ""))
+ .outputStrategy(OutputStrategies.append(EpProperties.stringEp(Labels.from("kvi", "Distance", ""),
+ "kvi", "http://kvi.de")))
+
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .build();
+ }
+
+ /*
+ TUTORIAL:
+ Here you get the Configuration Parameters which the User has entered
+ */
+ @Override
+ public ConfiguredEventProcessor<GoogleRoutingParameters> onInvocation(DataProcessorInvocation graph) {
+ ProcessingElementParameterExtractor extractor = getExtractor(graph);
+
+ String city = extractor.mappingPropertyValue(CITY_MAPPING);
+ String street = extractor.mappingPropertyValue(STREET_MAPPING);
+ String number = extractor.mappingPropertyValue(STREET_NUMBER_MAPPING);
+
+ String home = extractor.singleValueParameter(START_ADDRESS, String.class);
+
+ GoogleRoutingParameters params = new GoogleRoutingParameters(graph, city, street, number, home);
+ return new ConfiguredEventProcessor<>(params, () -> new GoogleRouting(params));
+ }
+}
diff --git a/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRoutingParameters.java b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRoutingParameters.java
new file mode 100644
index 0000000..fb88bc4
--- /dev/null
+++ b/streampipes-processors-geo-jvm/src/main/java/org/streampipes/processors/geo/jvm/processor/route/GoogleRoutingParameters.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.geo.jvm.processor.route;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class GoogleRoutingParameters extends EventProcessorBindingParams {
+ private String city;
+ private String street;
+ private String number;
+ private String home;
+
+ public GoogleRoutingParameters() {
+ }
+
+ public GoogleRoutingParameters(DataProcessorInvocation graph, String city, String street, String number, String home) {
+ super(graph);
+ this.city = city;
+ this.street = street;
+ this.number = number;
+ this.home = home;
+ }
+
+ public String getCity() {
+ return city;
+ }
+
+ public void setCity(String city) {
+ this.city = city;
+ }
+
+ public String getStreet() {
+ return street;
+ }
+
+ public void setStreet(String street) {
+ this.street = street;
+ }
+
+ public String getNumber() {
+ return number;
+ }
+
+ public void setNumber(String number) {
+ this.number = number;
+ }
+
+ public String getHome() {
+ return home;
+ }
+
+ public void setHome(String home) {
+ this.home = home;
+ }
+}
\ No newline at end of file
diff --git a/streampipes-processors-image-processing-jvm/Dockerfile b/streampipes-processors-image-processing-jvm/Dockerfile
new file mode 100644
index 0000000..3028bd7
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/Dockerfile
@@ -0,0 +1,8 @@
+FROM anapsix/alpine-java
+
+EXPOSE 8090
+ENV CONSUL_LOCATION consul
+
+ADD ./target/streampipes-processors-image-processing-jvm.jar /streampipes-processing-element-container.jar
+
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-image-processing-jvm/deployment/docker-compose.yml b/streampipes-processors-image-processing-jvm/deployment/docker-compose.yml
new file mode 100644
index 0000000..f5cffd0
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-image-processing-jvm:
+ image: ${SP_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-image-processing-jvm:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-image-processing-jvm/development/.env b/streampipes-processors-image-processing-jvm/development/.env
new file mode 100644
index 0000000..3c56349
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/development/.env
@@ -0,0 +1,9 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6035
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_JMS_HOST=localhost
+SP_NGINX_HOST=localhost
+SP_NGINX_PORT=8082
diff --git a/streampipes-processors-image-processing-jvm/pom.xml b/streampipes-processors-image-processing-jvm/pom.xml
new file mode 100644
index 0000000..4470e21
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/pom.xml
@@ -0,0 +1,106 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Copyright 2018 FZI Forschungszentrum Informatik
+ ~
+ ~ Licensed under the Apache License, Version 2.0 (the "License");
+ ~ you may not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-processors-image-processing-jvm</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-standalone</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-dataformat-json</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-messaging-kafka</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.boofcv</groupId>
+ <artifactId>boofcv-core</artifactId>
+ <version>0.29</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.streampipes.processors.imageprocessing.jvm.ImageProcessingJvmInit</mainClass>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
+ </exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>streampipes-processors-image-processing-jvm</finalName>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/ImageProcessingJvmInit.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/ImageProcessingJvmInit.java
new file mode 100644
index 0000000..2c5288d
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/ImageProcessingJvmInit.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.imageprocessing.jvm;
+
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.dataformat.json.JsonDataFormatFactory;
+import org.streampipes.messaging.jms.SpJmsProtocolFactory;
+import org.streampipes.messaging.kafka.SpKafkaProtocolFactory;
+import org.streampipes.processors.imageprocessing.jvm.config.ImageProcessingJvmConfig;
+import org.streampipes.processors.imageprocessing.jvm.processor.genericclassification.GenericImageClassificationController;
+import org.streampipes.processors.imageprocessing.jvm.processor.imagecropper.ImageCropperController;
+import org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment.ImageEnrichmentController;
+import org.streampipes.processors.imageprocessing.jvm.processor.qrreader.QrCodeReaderController;
+
+public class ImageProcessingJvmInit extends StandaloneModelSubmitter {
+
+ public static void main(String[] args) {
+ DeclarersSingleton
+ .getInstance()
+ .add(new ImageEnrichmentController())
+ .add(new ImageCropperController())
+ //.add(new ImageRectificationController())
+ .add(new QrCodeReaderController())
+ .add(new GenericImageClassificationController());
+
+ DeclarersSingleton.getInstance().registerDataFormat(new JsonDataFormatFactory());
+ DeclarersSingleton.getInstance().registerProtocol(new SpKafkaProtocolFactory());
+ DeclarersSingleton.getInstance().registerProtocol(new SpJmsProtocolFactory());
+
+ new ImageProcessingJvmInit().init(ImageProcessingJvmConfig.INSTANCE);
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/config/ConfigKeys.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/config/ConfigKeys.java
new file mode 100644
index 0000000..61f581d
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/config/ConfigKeys.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.imageprocessing.jvm.config;
+
+public class ConfigKeys {
+ final static String HOST = "SP_HOST";
+ final static String PORT = "SP_PORT";
+ final static String ICON_HOST = "SP_ICON_HOST";
+ final static String ICON_PORT = "SP_ICON_PORT";
+ final static String KAFKA_HOST = "SP_KAFKA_HOST";
+ final static String KAFKA_PORT = "SP_KAFKA_PORT";
+ final static String ZOOKEEPER_HOST = "SP_ZOOKEEPER_HOST";
+ final static String ZOOKEEPER_PORT = "SP_ZOOKEEPER_PORT";
+ final static String JMS_HOST = "SP_JMS_HOST";
+ final static String JMS_PORT = "SP_JMS_PORT";
+ final static String NGINX_HOST = "SP_NGINX_HOST";
+ final static String NGINX_PORT = "SP_NGINX_PORT";
+ final static String SERVICE_NAME_KEY = "SP_SERVICE_NAME";
+ final static String GOOGLE_API_KEY = "SP_GOOGLE_API_KEY";
+ final static String MODEL_DIRECTORY = "model_directory";
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/config/ImageProcessingJvmConfig.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/config/ImageProcessingJvmConfig.java
new file mode 100644
index 0000000..b7fd115
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/config/ImageProcessingJvmConfig.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.imageprocessing.jvm.config;
+
+
+import static org.streampipes.processors.imageprocessing.jvm.config.ConfigKeys.*;
+
+import org.streampipes.config.SpConfig;
+import org.streampipes.container.model.PeConfig;
+
+public enum ImageProcessingJvmConfig implements PeConfig {
+ INSTANCE;
+
+ private SpConfig config;
+
+ public final static String serverUrl;
+ public final static String iconBaseUrl;
+
+ private final static String service_id = "pe/org.streampipes.processors.imageprocessing.jvm";
+ private final static String service_name = "Processors Image Processing JVM";
+ private final static String service_container_name = "processors-image-processing-jvm";
+
+ ImageProcessingJvmConfig() {
+ config = SpConfig.getSpConfig(service_id);
+ config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe image processing");
+ config.register(ConfigKeys.PORT, 8090, "Port for the pe image processing");
+
+ config.register(ICON_HOST, "backend", "Hostname for the icon host");
+ config.register(ICON_PORT, 80, "Port for the icons in nginx");
+ config.register(ConfigKeys.NGINX_HOST, System.getenv("STREAMPIPES_HOST"), "External hostname of " +
+ "StreamPipes Nginx");
+ config.register(ConfigKeys.NGINX_PORT, 80, "External port of StreamPipes Nginx");
+ config.register(KAFKA_HOST, "kafka", "Host for kafka of the pe sinks project");
+ config.register(ConfigKeys.KAFKA_PORT, 9092, "Port for kafka of the pe sinks project");
+ config.register(ConfigKeys.ZOOKEEPER_HOST, "zookeeper", "Host for zookeeper of the pe sinks project");
+ config.register(ConfigKeys.ZOOKEEPER_PORT, 2181, "Port for zookeeper of the pe sinks project");
+ config.register(ConfigKeys.JMS_HOST, "tcp://activemq", "Hostname for pe actions service for active mq");
+ config.register(ConfigKeys.JMS_PORT, 61616, "Port for pe actions service for active mq");
+
+ config.register(MODEL_DIRECTORY, "/model-repository/", "The directory location for the folders of the image classification models");
+
+ config.register(ConfigKeys.SERVICE_NAME_KEY, service_name, "The name of the service");
+
+ }
+
+ static {
+ serverUrl = ImageProcessingJvmConfig.INSTANCE.getHost() + ":" + ImageProcessingJvmConfig.INSTANCE.getPort();
+ iconBaseUrl = "http://" +ImageProcessingJvmConfig.INSTANCE.getIconHost() + ":" +
+ ImageProcessingJvmConfig.INSTANCE.getIconPort() +"/assets/img/pe_icons";
+ }
+
+ public static final String getIconUrl(String pictureName) {
+ return iconBaseUrl +"/" +pictureName +".png";
+ }
+
+ @Override
+ public String getHost() {
+ return config.getString(ConfigKeys.HOST);
+ }
+
+ @Override
+ public int getPort() {
+ return config.getInteger(ConfigKeys.PORT);
+ }
+
+ public String getIconHost() {
+ return config.getString(ICON_HOST);
+ }
+
+ public int getIconPort() {
+ return config.getInteger(ICON_PORT);
+ }
+
+ public String getKafkaHost() {
+ return config.getString(KAFKA_HOST);
+ }
+
+ public int getKafkaPort() {
+ return config.getInteger(KAFKA_PORT);
+ }
+
+ public String getKafkaUrl() {
+ return getKafkaHost() + ":" + getKafkaPort();
+ }
+
+ public String getZookeeperHost() {
+ return config.getString(ZOOKEEPER_HOST);
+ }
+
+ public int getZookeeperPort() {
+ return config.getInteger(ZOOKEEPER_PORT);
+ }
+
+ public String getJmsHost() {
+ return config.getString(JMS_HOST);
+ }
+
+ public int getJmsPort() {
+ return config.getInteger(JMS_PORT);
+ }
+
+ public String getJmsUrl() {
+ return getJmsHost() + ":" + getJmsPort();
+ }
+
+ public String getNginxHost() {
+ return config.getString(NGINX_HOST);
+ }
+
+ public Integer getNginxPort() {
+ return config.getInteger(NGINX_PORT);
+ }
+
+ public String getModelDirectory() {
+ return config.getString(MODEL_DIRECTORY);
+ }
+
+ @Override
+ public String getId() {
+ return service_id;
+ }
+
+ @Override
+ public String getName() {
+ return config.getString(SERVICE_NAME_KEY);
+ }
+
+
+
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/ImageTransformer.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/ImageTransformer.java
new file mode 100644
index 0000000..e8a5c97
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/ImageTransformer.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.commons;
+
+import org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment.BoxCoordinates;
+import org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment.ImageEnrichmentParameters;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+
+public class ImageTransformer extends PlainImageTransformer<ImageEnrichmentParameters> {
+
+ public ImageTransformer(Map<String, Object> in, ImageEnrichmentParameters params) {
+ super(in, params);
+ }
+
+ public Optional<BufferedImage> getImage() {
+
+ return getImage(params.getImageProperty());
+ }
+
+ public BoxCoordinates getBoxCoordinates(BufferedImage image) {
+ Float x = toFloat(in.get(params.getBoxX()));
+ Float y = toFloat(in.get(params.getBoxY()));
+ Float width = toFloat(in.get(params.getBoxWidth()));
+ Float height = toFloat(in.get(params.getBoxHeight()));
+
+ return BoxCoordinates.make(image.getWidth(), image.getHeight(), width, height, x, y);
+ }
+
+ public Optional<byte[]> makeImage(BufferedImage image) {
+
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ImageIO.write(image, "jpg", baos);
+ baos.flush();
+ byte[] finalImage = baos.toByteArray();
+ baos.close();
+ return Optional.of(finalImage);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return Optional.empty();
+ }
+
+ }
+
+ private Float toFloat(Object obj) {
+ return Float.parseFloat(String.valueOf(obj));
+ }
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/PlainImageTransformer.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/PlainImageTransformer.java
new file mode 100644
index 0000000..19be260
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/PlainImageTransformer.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.commons;
+
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+import javax.imageio.ImageIO;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Base64;
+import java.util.Map;
+import java.util.Optional;
+
+public class PlainImageTransformer<T extends EventProcessorBindingParams> {
+
+ protected Map<String, Object> in;
+ protected T params;
+
+ public PlainImageTransformer(Map<String, Object> in, T params) {
+ this.in = in;
+ this.params = params;
+ }
+
+ public Optional<BufferedImage> getImage(String imagePropertyName) {
+ System.out.println(imagePropertyName);
+ System.out.println(in.get(imagePropertyName));
+ String imageBase64 = String.valueOf(in.get(imagePropertyName));
+
+ InputStream img = new ByteArrayInputStream(Base64.getDecoder().decode(imageBase64));
+ try {
+ return Optional.of(ImageIO.read(img));
+ } catch (IOException e) {
+ e.printStackTrace();
+ return Optional.empty();
+ }
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/RequiredBoxStream.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/RequiredBoxStream.java
new file mode 100644
index 0000000..5ab8040
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/commons/RequiredBoxStream.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.commons;
+
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.helpers.CollectedStreamRequirements;
+import org.streampipes.sdk.helpers.EpRequirements;
+import org.streampipes.sdk.helpers.Labels;
+
+public class RequiredBoxStream {
+
+ public static final String IMAGE_PROPERTY = "image-property";
+ public static final String BOX_WIDTH_PROPERTY = "box-width-property";
+ public static final String BOX_HEIGHT_PROPERTY = "box-height-property";
+ public static final String BOX_X_PROPERTY = "box-x-property";
+ public static final String BOX_Y_PROPERTY = "box-y-property";
+
+ public static CollectedStreamRequirements getBoxStream() {
+
+ return StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq("https://image.com"), Labels
+ .from(IMAGE_PROPERTY, "Image Classification", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq("https://schema.org/width"),
+ Labels.from(BOX_WIDTH_PROPERTY, "Box Width", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq("https://schema.org/height"),
+ Labels.from(BOX_HEIGHT_PROPERTY, "Box Height", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq("https://schema.org/x"),
+ Labels.from(BOX_X_PROPERTY, "Box X Coordinate", ""),
+ PropertyScope.NONE)
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq("https://schema.org/y"),
+ Labels.from(BOX_Y_PROPERTY, "Box Y Coordinate", ""),
+ PropertyScope.NONE)
+ .build();
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassification.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassification.java
new file mode 100644
index 0000000..d97cbe6
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassification.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.genericclassification;
+
+import boofcv.abst.scene.ImageClassifier;
+import boofcv.factory.scene.ClassifierAndSource;
+import boofcv.factory.scene.FactoryImageClassifier;
+import boofcv.io.image.ConvertBufferedImage;
+import boofcv.struct.image.GrayF32;
+import boofcv.struct.image.Planar;
+import deepboof.io.DeepBoofDataBaseOps;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.imageprocessing.jvm.processor.commons.PlainImageTransformer;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.IOException;
+import java.util.*;
+
+public class GenericImageClassification extends StandaloneEventProcessorEngine<GenericImageClassificationParameters> {
+
+ private GenericImageClassificationParameters params;
+ private ClassifierAndSource cs;
+
+ private ImageClassifier<Planar<GrayF32>> classifier;
+ private List<String> categories;
+
+ public GenericImageClassification(GenericImageClassificationParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(GenericImageClassificationParameters genericImageClassificationParameters, DataProcessorInvocation dataProcessorInvocation) {
+ this.params = genericImageClassificationParameters;
+ //this.cs = FactoryImageClassifier.vgg_cifar10(); // Test set 89.9% for 10 categories
+ ClassifierAndSource cs = FactoryImageClassifier.nin_imagenet(); // Test set 62.6% for 1000 categories
+
+ File path = DeepBoofDataBaseOps.downloadModel(cs.getSource(), new File("download_data"));
+
+ this.classifier = cs.getClassifier();
+ try {
+ this.classifier.loadModel(path);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ this.categories = classifier.getCategories();
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> in, String s, SpOutputCollector out) {
+ PlainImageTransformer<GenericImageClassificationParameters> imageTransformer = new PlainImageTransformer<>(in,
+ params);
+
+
+ Optional<BufferedImage> imageOpt = imageTransformer.getImage(params.getImagePropertyName());
+ if (imageOpt.isPresent()) {
+ BufferedImage buffered = imageOpt.get();
+ Planar<GrayF32> image = new Planar<>(GrayF32.class, buffered.getWidth(), buffered.getHeight(), 3);
+ ConvertBufferedImage.convertFromPlanar(buffered, image, true, GrayF32.class);
+
+ classifier.classify(image);
+ List<ImageClassifier.Score> scores = classifier.getAllResults();
+ scores.sort(new Comparator<ImageClassifier.Score>() {
+ @Override
+ public int compare(ImageClassifier.Score o1, ImageClassifier.Score o2) {
+ return (o1.score - o2.score) >= 0 ? -1 : 1;
+ }
+ });
+ //Collections.reverse(scores);
+
+ if (scores.size() > 0) {
+ System.out.println(scores.get(0).score +":" +categories.get(scores.get(0).category));
+ //scores.forEach(score -> System.out.println(score.category +":" +categories.get(score.category) +":" +score));
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("score", scores.get(0).score);
+ outMap.put("category", categories.get(scores.get(0).category));
+ out.onEvent(outMap);
+ }
+ }
+ }
+
+ @Override
+ public void onDetach() {
+
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassificationController.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassificationController.java
new file mode 100644
index 0000000..01fb768
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassificationController.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.genericclassification;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+public class GenericImageClassificationController extends StandaloneEventProcessingDeclarer<GenericImageClassificationParameters> {
+
+ private static final String IMAGE = "IMAGE";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processor.imageclassification.jvm.generic-image-classification", "Generic Image Classification", "Image " +
+ "Classification Description (Generic Model)")
+ .category(DataProcessorType.FILTER)
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements
+ .domainPropertyReq("https://image.com"), Labels.from(IMAGE, "Image Classification", ""),
+ PropertyScope.NONE)
+ .build())
+ .outputStrategy(OutputStrategies.fixed(
+ EpProperties.doubleEp(Labels.empty(), "score", "https://schema.org/score"),
+ EpProperties.stringEp(Labels.empty(), "category", "https://schema.org/category")
+
+ ))
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+
+ @Override
+ public ConfiguredEventProcessor<GenericImageClassificationParameters> onInvocation(DataProcessorInvocation graph) {
+ ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(graph);
+
+ String imageProperty = extractor.mappingPropertyValue(IMAGE);
+
+ GenericImageClassificationParameters staticParam = new GenericImageClassificationParameters(graph, imageProperty);
+
+ return new ConfiguredEventProcessor<>(staticParam, () -> new GenericImageClassification(staticParam));
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassificationParameters.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassificationParameters.java
new file mode 100644
index 0000000..2d4a54f
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/genericclassification/GenericImageClassificationParameters.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.genericclassification;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class GenericImageClassificationParameters extends EventProcessorBindingParams {
+
+ private String imagePropertyName;
+
+ public GenericImageClassificationParameters(DataProcessorInvocation graph, String imagePropertyName) {
+ super(graph);
+ this.imagePropertyName = imagePropertyName;
+ }
+
+ public String getImagePropertyName() {
+ return imagePropertyName;
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropper.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropper.java
new file mode 100644
index 0000000..3f15079
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropper.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imagecropper;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.imageprocessing.jvm.processor.commons.ImageTransformer;
+import org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment.BoxCoordinates;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import java.awt.image.BufferedImage;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public class ImageCropper extends StandaloneEventProcessorEngine<ImageCropperParameters> {
+
+ private ImageCropperParameters params;
+
+ public ImageCropper(ImageCropperParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(ImageCropperParameters imageCropperParameters, DataProcessorInvocation dataProcessorInvocation) {
+ this.params = imageCropperParameters;
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> in, String s, SpOutputCollector out) {
+ ImageTransformer imageTransformer = new ImageTransformer(in, params);
+ Optional<BufferedImage> imageOpt = imageTransformer.getImage();
+
+ if (imageOpt.isPresent()) {
+ BufferedImage image = imageOpt.get();
+ BoxCoordinates boxCoordinates = imageTransformer.getBoxCoordinates(image);
+
+ BufferedImage dest = image.getSubimage(boxCoordinates.getX(), boxCoordinates.getY(), boxCoordinates.getWidth(),
+ boxCoordinates.getHeight());
+
+ Optional<byte[]> finalImage = imageTransformer.makeImage(dest);
+
+ if (finalImage.isPresent()) {
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("image", Base64.getEncoder().encodeToString(finalImage.get()));
+ out.onEvent(outMap);
+ }
+ }
+ }
+
+ @Override
+ public void onDetach() {
+
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropperController.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropperController.java
new file mode 100644
index 0000000..dd89bed
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropperController.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imagecropper;
+
+
+import static org.streampipes.processors.imageprocessing.jvm.processor.commons.RequiredBoxStream.*;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.imageprocessing.jvm.config.ImageProcessingJvmConfig;
+import org.streampipes.processors.imageprocessing.jvm.processor.commons.RequiredBoxStream;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.EpProperties;
+import org.streampipes.sdk.helpers.Labels;
+import org.streampipes.sdk.helpers.OutputStrategies;
+import org.streampipes.sdk.helpers.SupportedFormats;
+import org.streampipes.sdk.helpers.SupportedProtocols;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+
+public class ImageCropperController extends StandaloneEventProcessingDeclarer<ImageCropperParameters> {
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processor.imageclassification.jvm.image-cropper", "Image Cropper", "Image Enrichment: Crops an " +
+ "image based on " +
+ "given bounding box coordinates")
+ .iconUrl(ImageProcessingJvmConfig.getIconUrl( "crop"))
+ .category(DataProcessorType.FILTER)
+ .requiredStream(RequiredBoxStream.getBoxStream())
+ .outputStrategy(OutputStrategies.fixed(
+ EpProperties.stringEp(Labels.empty(), "image", "https://image.com")
+
+ ))
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+
+ @Override
+ public ConfiguredEventProcessor<ImageCropperParameters> onInvocation(DataProcessorInvocation dataProcessorInvocation) {
+ ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(dataProcessorInvocation);
+
+ String imageProperty = extractor.mappingPropertyValue(IMAGE_PROPERTY);
+ String boxWidthProperty = extractor.mappingPropertyValue(BOX_WIDTH_PROPERTY);
+ String boxHeightProperty = extractor.mappingPropertyValue(BOX_HEIGHT_PROPERTY);
+ String boxXProperty = extractor.mappingPropertyValue(BOX_X_PROPERTY);
+ String boxYProperty = extractor.mappingPropertyValue(BOX_Y_PROPERTY);
+
+ ImageCropperParameters params = new ImageCropperParameters(dataProcessorInvocation, imageProperty,
+ boxWidthProperty, boxHeightProperty, boxXProperty, boxYProperty);
+
+ return new ConfiguredEventProcessor<>(params, () -> new ImageCropper(params));
+ }
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropperParameters.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropperParameters.java
new file mode 100644
index 0000000..53cbc41
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagecropper/ImageCropperParameters.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imagecropper;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment.ImageEnrichmentParameters;
+
+public class ImageCropperParameters extends ImageEnrichmentParameters {
+
+ public ImageCropperParameters(DataProcessorInvocation graph, String imageProperty, String boxWidth, String boxHeight, String boxX, String boxY) {
+ super(graph, imageProperty, boxWidth, boxHeight, boxX, boxY);
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/BoxCoordinates.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/BoxCoordinates.java
new file mode 100644
index 0000000..d2a1f33
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/BoxCoordinates.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment;
+
+public class BoxCoordinates {
+
+ private int width;
+ private int height;
+ private int x;
+ private int y;
+
+ public static BoxCoordinates make(int originalWidth, int originalHeight, Float width, Float height, Float x, Float
+ y) {
+ return new BoxCoordinates(Math.round(width * originalWidth),
+ Math.round(height * originalHeight),
+ Math.round(x * originalWidth),
+ Math.round(y* originalHeight));
+ }
+
+ public BoxCoordinates(int width, int height, int x, int y) {
+ this.width = width;
+ this.height = height;
+ this.x = x;
+ this.y = y;
+ }
+
+ public int getWidth() {
+ return width;
+ }
+
+ public int getHeight() {
+ return height;
+ }
+
+ public int getX() {
+ return x;
+ }
+
+ public int getY() {
+ return y;
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnricher.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnricher.java
new file mode 100644
index 0000000..baac026
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnricher.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import javax.imageio.ImageIO;
+import java.awt.*;
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.*;
+import java.util.List;
+
+public class ImageEnricher extends StandaloneEventProcessorEngine<ImageEnrichmentParameters> {
+
+ private ImageEnrichmentParameters params;
+
+ public ImageEnricher(ImageEnrichmentParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(ImageEnrichmentParameters params, DataProcessorInvocation graph) {
+ this.params = params;
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> in, String s, SpOutputCollector out) {
+
+ List<Map<String, Object>> allBoxes = (List<Map<String, Object>>) in.get(params.getBoxArray());
+
+ Optional<BufferedImage> imageOpt = getImage(in.get(params.getImageProperty()));
+
+ if (imageOpt.isPresent()) {
+ BufferedImage image = imageOpt.get();
+
+ for (Map<String, Object> box : allBoxes) {
+//
+ BoxCoordinates boxCoordinates = getBoxCoordinates(image, box);
+
+ Graphics2D graph = image.createGraphics();
+ int alpha = 180;
+ Color color = new Color(133, 148, 229, alpha);
+ graph.setColor(color);
+ graph.fill(new Rectangle(boxCoordinates.getX(), boxCoordinates.getY(), boxCoordinates.getWidth(),
+ boxCoordinates.getHeight()));
+ graph.dispose();
+
+ }
+
+ // TODO howto change final image
+ Optional<byte[]> finalImage = makeImage(image);
+
+ if (finalImage.isPresent()) {
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("image", Base64.getEncoder().encodeToString(finalImage.get()));
+ out.onEvent(outMap);
+ }
+ }
+
+ }
+
+ private BoxCoordinates getBoxCoordinates(BufferedImage image, Map<String, Object> box) {
+ Float x = toFloat(box.get(params.getBoxX()));
+ Float y = toFloat(box.get(params.getBoxY()));
+ Float width = toFloat(box.get(params.getBoxWidth()));
+ Float height = toFloat(box.get(params.getBoxHeight()));
+
+ return BoxCoordinates.make(image.getWidth(), image.getHeight(), width, height, x, y);
+ }
+
+ private Float toFloat(Object obj) {
+ return Float.parseFloat(String.valueOf(obj));
+ }
+
+
+ private Optional<byte[]> makeImage(BufferedImage image) {
+
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ImageIO.write(image, "jpg", baos);
+ baos.flush();
+ byte[] finalImage = baos.toByteArray();
+ baos.close();
+ return Optional.of(finalImage);
+ } catch (IOException e) {
+ e.printStackTrace();
+ return Optional.empty();
+ }
+
+ }
+
+
+ private Optional<BufferedImage> getImage(Object image) {
+ String imageBase64 = String.valueOf(image);
+
+ InputStream img = new ByteArrayInputStream(Base64.getDecoder().decode(imageBase64));
+ try {
+ return Optional.of(ImageIO.read(img));
+ } catch (IOException e) {
+ e.printStackTrace();
+ return Optional.empty();
+ }
+ }
+ @Override
+ public void onDetach() {
+
+ }
+
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnrichmentController.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnrichmentController.java
new file mode 100644
index 0000000..cb99934
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnrichmentController.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment;
+
+import static org.streampipes.processors.imageprocessing.jvm.processor.commons.RequiredBoxStream.IMAGE_PROPERTY;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.imageprocessing.jvm.config.ImageProcessingJvmConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.CollectedStreamRequirements;
+import org.streampipes.sdk.helpers.EpProperties;
+import org.streampipes.sdk.helpers.EpRequirements;
+import org.streampipes.sdk.helpers.Labels;
+import org.streampipes.sdk.helpers.OutputStrategies;
+import org.streampipes.sdk.helpers.SupportedFormats;
+import org.streampipes.sdk.helpers.SupportedProtocols;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+public class ImageEnrichmentController extends StandaloneEventProcessingDeclarer<ImageEnrichmentParameters> {
+
+ public static final String BOX_ARRAY_PROPERTY = "box-array-property";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processor.imageclassification.jvm.image-enricher", "Image Enricher", "Image Enrichment: Enriches an " +
+ "image with " +
+ "given bounding box coordinates")
+ .iconUrl(ImageProcessingJvmConfig.getIconUrl( "image_enrich"))
+ .category(DataProcessorType.FILTER)
+ .requiredStream(getStreamRequirements())
+
+ .outputStrategy(OutputStrategies.fixed(
+ EpProperties.stringEp(Labels.empty(), "image", "https://image.com")
+
+ ))
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+
+ private CollectedStreamRequirements getStreamRequirements() {
+ return StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReq("https://image.com"), Labels
+ .from(IMAGE_PROPERTY, "Image Classification", ""),
+ PropertyScope.NONE)
+ // TODO add again
+ .requiredPropertyWithUnaryMapping(EpRequirements.domainPropertyReqList("https://streampipes.org/boundingboxes"),
+ Labels.from(BOX_ARRAY_PROPERTY, "Array Width Bounding Boxes", "Contains an array with bounding boxes"),
+ PropertyScope.NONE)
+ .build();
+ }
+
+ @Override
+ public ConfiguredEventProcessor<ImageEnrichmentParameters> onInvocation(DataProcessorInvocation dataProcessorInvocation) {
+ ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(dataProcessorInvocation);
+
+ String imageProperty = extractor.mappingPropertyValue(IMAGE_PROPERTY);
+ String boxArray = extractor.mappingPropertyValue(BOX_ARRAY_PROPERTY);
+// String boxArray = "boxes";
+
+ ImageEnrichmentParameters params = new ImageEnrichmentParameters(dataProcessorInvocation, imageProperty,
+ boxArray, "box_width", "box_height", "box_x", "box_y");
+
+ return new ConfiguredEventProcessor<>(params, () -> new ImageEnricher(params));
+
+
+ }
+
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnrichmentParameters.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnrichmentParameters.java
new file mode 100644
index 0000000..8bb30b7
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imageenrichment/ImageEnrichmentParameters.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imageenrichment;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class ImageEnrichmentParameters extends EventProcessorBindingParams {
+
+ private String imageProperty;
+ private String boxArray;
+
+ private String boxWidth;
+ private String boxHeight;
+ private String boxX;
+ private String boxY;
+
+ public ImageEnrichmentParameters(DataProcessorInvocation graph, String imageProperty, String boxArray, String boxWidth, String boxHeight, String boxX, String boxY) {
+ super(graph);
+ this.imageProperty = imageProperty;
+ this.boxArray = boxArray;
+ this.boxWidth = boxWidth;
+ this.boxHeight = boxHeight;
+ this.boxX = boxX;
+ this.boxY = boxY;
+ }
+
+
+ public ImageEnrichmentParameters(DataProcessorInvocation graph, String imageProperty, String boxWidth, String boxHeight, String boxX, String boxY) {
+ super(graph);
+ this.imageProperty = imageProperty;
+ this.boxWidth = boxWidth;
+ this.boxHeight = boxHeight;
+ this.boxX = boxX;
+ this.boxY = boxY;
+ }
+
+ public String getImageProperty() {
+ return imageProperty;
+ }
+
+ public String getBoxArray() {
+ return boxArray;
+ }
+
+ public String getBoxWidth() {
+ return boxWidth;
+ }
+
+ public String getBoxHeight() {
+ return boxHeight;
+ }
+
+ public String getBoxX() {
+ return boxX;
+ }
+
+ public String getBoxY() {
+ return boxY;
+ }
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectificationController.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectificationController.java
new file mode 100644
index 0000000..b3b6bc9
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectificationController.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imagerectification;
+
+import static org.streampipes.processors.imageprocessing.jvm.processor.commons.RequiredBoxStream.IMAGE_PROPERTY;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.EpRequirements;
+import org.streampipes.sdk.helpers.Labels;
+import org.streampipes.sdk.helpers.OutputStrategies;
+import org.streampipes.sdk.helpers.SupportedFormats;
+import org.streampipes.sdk.helpers.SupportedProtocols;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+public class ImageRectificationController extends StandaloneEventProcessingDeclarer<ImageRectificationParameters> {
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processor.imageclassification.image-rectifier", "Image Rectifier", "Image Rectification: Rectifies " +
+ "an image")
+ .category(DataProcessorType.FILTER)
+ .requiredStream(StreamRequirementsBuilder.create().requiredPropertyWithUnaryMapping(EpRequirements
+ .domainPropertyReq("https://image.com"), Labels
+ .from(IMAGE_PROPERTY, "Image Classification", ""),
+ PropertyScope.NONE).build())
+ .outputStrategy(OutputStrategies.keep())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+
+ @Override
+ public ConfiguredEventProcessor<ImageRectificationParameters> onInvocation(DataProcessorInvocation dataProcessorInvocation) {
+ ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(dataProcessorInvocation);
+
+ String imagePropertyName = extractor.mappingPropertyValue(IMAGE_PROPERTY);
+
+ ImageRectificationParameters params = new ImageRectificationParameters(dataProcessorInvocation, imagePropertyName);
+
+ return new ConfiguredEventProcessor<>(params, () -> new ImageRectifier(params));
+ }
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectificationParameters.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectificationParameters.java
new file mode 100644
index 0000000..28ef98a
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectificationParameters.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imagerectification;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class ImageRectificationParameters extends EventProcessorBindingParams {
+
+ private String imageProperty;
+
+ public ImageRectificationParameters(DataProcessorInvocation graph, String imageProperty) {
+ super(graph);
+ this.imageProperty = imageProperty;
+ }
+
+ public String getImageProperty() {
+ return imageProperty;
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectifier.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectifier.java
new file mode 100644
index 0000000..ed664be
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/imagerectification/ImageRectifier.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.imagerectification;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import java.util.Map;
+
+public class ImageRectifier extends StandaloneEventProcessorEngine<ImageRectificationParameters> {
+
+ private ImageRectificationParameters params;
+
+ public ImageRectifier(ImageRectificationParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(ImageRectificationParameters imageRectificationParameters, DataProcessorInvocation dataProcessorInvocation) {
+ this.params = imageRectificationParameters;
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> map, String s, SpOutputCollector spOutputCollector) {
+ // TODO add logic here
+ }
+
+ @Override
+ public void onDetach() {
+
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReader.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReader.java
new file mode 100644
index 0000000..78db6d5
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReader.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.qrreader;
+
+import boofcv.abst.fiducial.QrCodeDetector;
+import boofcv.alg.fiducial.qrcode.QrCode;
+import boofcv.factory.fiducial.FactoryFiducial;
+import boofcv.io.image.ConvertBufferedImage;
+import boofcv.struct.image.GrayU8;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.imageprocessing.jvm.processor.commons.PlainImageTransformer;
+import org.streampipes.wrapper.routing.SpOutputCollector;
+import org.streampipes.wrapper.standalone.engine.StandaloneEventProcessorEngine;
+
+import java.awt.image.BufferedImage;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class QrCodeReader extends StandaloneEventProcessorEngine<QrCodeReaderParameters> {
+
+ private QrCodeReaderParameters params;
+ private static final Logger LOG = LoggerFactory.getLogger(QrCodeReader.class);
+
+ public QrCodeReader(QrCodeReaderParameters params) {
+ super(params);
+ }
+
+ @Override
+ public void onInvocation(QrCodeReaderParameters qrCodeReaderParameters, DataProcessorInvocation dataProcessorInvocation) {
+ this.params = qrCodeReaderParameters;
+ }
+
+ @Override
+ public void onEvent(Map<String, Object> in, String s, SpOutputCollector out) {
+ PlainImageTransformer<QrCodeReaderParameters> imageTransformer = new PlainImageTransformer<>(in, params);
+ Optional<BufferedImage> imageOpt = imageTransformer.getImage(params.getImagePropertyName());
+
+ if (imageOpt.isPresent()) {
+ BufferedImage input = imageOpt.get();
+
+ GrayU8 gray = ConvertBufferedImage.convertFrom(input,(GrayU8)null);
+
+ QrCodeDetector<GrayU8> detector = FactoryFiducial.qrcode(null,GrayU8.class);
+
+ detector.process(gray);
+ List<QrCode> detections = detector.getDetections();
+
+ if (detections.size() > 0) {
+ LOG.info(detections.get(0).message);
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("qrvalue", detections.get(0).message);
+ outMap.put("timestamp", System.currentTimeMillis());
+ out.onEvent(outMap);
+ } else {
+ LOG.info("Could not find any QR code");
+ }
+
+
+ }
+ }
+
+ @Override
+ public void onDetach() {
+
+ }
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReaderController.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReaderController.java
new file mode 100644
index 0000000..a0aa94d
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReaderController.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.qrreader;
+
+import static org.streampipes.processors.imageprocessing.jvm.processor.commons.RequiredBoxStream.IMAGE_PROPERTY;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.imageprocessing.jvm.config.ImageProcessingJvmConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.EpProperties;
+import org.streampipes.sdk.helpers.EpRequirements;
+import org.streampipes.sdk.helpers.Labels;
+import org.streampipes.sdk.helpers.OutputStrategies;
+import org.streampipes.sdk.helpers.SupportedFormats;
+import org.streampipes.sdk.helpers.SupportedProtocols;
+import org.streampipes.wrapper.standalone.ConfiguredEventProcessor;
+import org.streampipes.wrapper.standalone.declarer.StandaloneEventProcessingDeclarer;
+
+public class QrCodeReaderController extends StandaloneEventProcessingDeclarer<QrCodeReaderParameters> {
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("qr-code-reader", "QR Code Reader", ("QR Code Reader: Detects a QR Code " +
+ "in an image"))
+ .category(DataProcessorType.FILTER)
+ .iconUrl(ImageProcessingJvmConfig.getIconUrl("qrcode"))
+
+ .requiredStream(StreamRequirementsBuilder.create().requiredPropertyWithUnaryMapping(EpRequirements
+ .domainPropertyReq("https://image.com"), Labels
+ .from(IMAGE_PROPERTY, "Image", ""),
+ PropertyScope.NONE).build())
+ .outputStrategy(OutputStrategies.fixed(EpProperties.timestampProperty("timestamp"),
+ EpProperties.stringEp(Labels.from("qr-value", "QR code value", ""),
+ "qrvalue", "http://schema.org/text")))
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+
+ @Override
+ public ConfiguredEventProcessor<QrCodeReaderParameters> onInvocation(DataProcessorInvocation dataProcessorInvocation) {
+ ProcessingElementParameterExtractor extractor = ProcessingElementParameterExtractor.from(dataProcessorInvocation);
+
+ String imagePropertyName = extractor.mappingPropertyValue(IMAGE_PROPERTY);
+
+ QrCodeReaderParameters params = new QrCodeReaderParameters(dataProcessorInvocation, imagePropertyName);
+
+ return new ConfiguredEventProcessor<>(params, () -> new QrCodeReader(params));
+ }
+
+}
diff --git a/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReaderParameters.java b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReaderParameters.java
new file mode 100644
index 0000000..30c056f
--- /dev/null
+++ b/streampipes-processors-image-processing-jvm/src/main/java/org/streampipes/processors/imageprocessing/jvm/processor/qrreader/QrCodeReaderParameters.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.imageprocessing.jvm.processor.qrreader;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class QrCodeReaderParameters extends EventProcessorBindingParams {
+
+ private String imagePropertyName;
+
+ public QrCodeReaderParameters(DataProcessorInvocation graph, String imagePropertyName) {
+ super(graph);
+ this.imagePropertyName = imagePropertyName;
+ }
+
+ public String getImagePropertyName() {
+ return imagePropertyName;
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/Dockerfile b/streampipes-processors-pattern-detection-flink/Dockerfile
index 56c0f40..4ea19b3 100644
--- a/streampipes-processors-pattern-detection-flink/Dockerfile
+++ b/streampipes-processors-pattern-detection-flink/Dockerfile
@@ -3,6 +3,6 @@
EXPOSE 8090
ENV CONSUL_LOCATION consul
-ADD ./target/streampipes-processors-pattern-detection-flink.jar /streampipes-processors-pattern-detection-flink.jar
+ADD ./target/streampipes-processors-pattern-detection-flink.jar /streampipes-processing-element-container.jar
-ENTRYPOINT ["java", "-jar", "/streampipes-processors-pattern-detection-flink.jar"]
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-pattern-detection-flink/deployment/docker-compose.yml b/streampipes-processors-pattern-detection-flink/deployment/docker-compose.yml
new file mode 100644
index 0000000..57a9d23
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-pattern-detection-flink:
+ image: ${SP_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-pattern-detection-flink:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-pattern-detection-flink/development/.env b/streampipes-processors-pattern-detection-flink/development/.env
new file mode 100644
index 0000000..a0c8a7b
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/development/.env
@@ -0,0 +1,8 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6040
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_FLINK_DEBUG=true
+SP_ELASTICSEARCH_HOST=localhost
diff --git a/streampipes-processors-pattern-detection-flink/pom.xml b/streampipes-processors-pattern-detection-flink/pom.xml
index f80cc96..8d86c94 100644
--- a/streampipes-processors-pattern-detection-flink/pom.xml
+++ b/streampipes-processors-pattern-detection-flink/pom.xml
@@ -1,18 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>streampipes-pipeline-elements</artifactId>
<groupId>org.streampipes</groupId>
- <version>0.55.3-SNAPSHOT</version>
+ <version>0.60.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>streampipes-processors-pattern-detection-flink</artifactId>
<properties>
- <streampipes.version>0.55.3-SNAPSHOT</streampipes.version>
<elasticsearch.version>5.2.2</elasticsearch.version>
</properties>
@@ -20,7 +17,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-container-standalone</artifactId>
- <version>${streampipes.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.kafka</groupId>
@@ -35,7 +31,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-commons</artifactId>
- <version>${streampipes.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.kafka</groupId>
@@ -50,7 +45,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-wrapper-flink</artifactId>
- <version>${streampipes.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
@@ -70,33 +64,6 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-sdk</artifactId>
- <version>${streampipes.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-filesystem_2.11</artifactId>
- <version>1.4.0</version>
- </dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <version>3.6.1</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch.client</groupId>
- <artifactId>elasticsearch-rest-high-level-client</artifactId>
- <version>6.2.3</version>
- </dependency>
- <dependency>
- <groupId>org.elasticsearch</groupId>
- <artifactId>elasticsearch</artifactId>
- <version>6.2.3</version>
- <exclusions>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
@@ -111,58 +78,35 @@
<dependency>
<groupId>org.streampipes</groupId>
<artifactId>streampipes-config</artifactId>
- <version>${streampipes.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-cep-scala_2.11</artifactId>
+ <artifactId>flink-cep_2.11</artifactId>
<version>1.4.0</version>
</dependency>
+
+ <dependency>
+ <groupId>io.flinkspector</groupId>
+ <artifactId>flinkspector-datastream_2.11</artifactId>
+ <version>0.8.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
- <pluginManagement>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>3.0.0</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>3.1</version>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- <encoding>UTF-8</encoding>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.10.4</version>
- </plugin>
- </plugins>
- </pluginManagement>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <additionalparam>-Xdoclint:none</additionalparam>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.3</version>
<executions>
@@ -200,85 +144,9 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>external.atlassian.jgitflow</groupId>
- <artifactId>jgitflow-maven-plugin</artifactId>
- <version>1.0-m5.1</version>
- <configuration>
- <flowInitContext>
- <masterBranchName>master</masterBranchName>
- <developBranchName>dev</developBranchName>
- <featureBranchPrefix>feature-</featureBranchPrefix>
- <releaseBranchPrefix>release-</releaseBranchPrefix>
- <hotfixBranchPrefix>hotfix-</hotfixBranchPrefix>
- <versionTagPrefix>version-</versionTagPrefix>
- </flowInitContext>
- <noDeploy>false</noDeploy>
- <autoVersionSubmodules>true</autoVersionSubmodules>
- <pushReleases>false</pushReleases>
- <localOnly>true</localOnly>
- <squash>false</squash>
- <scmCommentPrefix>[RELEASE] [skip-ci]</scmCommentPrefix>
- <enableSshAgent>true</enableSshAgent>
- </configuration>
- </plugin>
</plugins>
<finalName>streampipes-processors-pattern-detection-flink</finalName>
</build>
- <scm>
- <developerConnection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-flink.git
- </developerConnection>
- <connection>scm:git:ssh://git@ipe-wim-gitlab.fzi.de:2222/streampipes/pe-examples-flink.git</connection>
- <url>https://github.com/streampipes/pe-examples-flink</url>
- </scm>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
-
- <developers>
- <developer>
- <name>Dominik Riemer</name>
- <email>riemer@fzi.de</email>
- </developer>
- <developer>
- <name>Philipp Zehnder</name>
- <email>zehnder@fzi.de</email>
- </developer>
- </developers>
-
- <repositories>
- <repository>
- <id>laus</id>
- <name>nexus repository</name>
- <url>https://laus.fzi.de/nexus/content/repositories/public/</url>
- <releases>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </releases>
- <snapshots>
- <enabled>true</enabled>
- <updatePolicy>always</updatePolicy>
- </snapshots>
- </repository>
- </repositories>
-
- <distributionManagement>
- <repository>
- <id>sonatype</id>
- <name>Releases</name>
- <url>https://oss.sonatype.org/service/local/staging/deploy/maven2</url>
- </repository>
- <snapshotRepository>
- <id>deployment</id>
- <name>Internal Releases</name>
- <url>https://laus.fzi.de/nexus/content/repositories/snapshots/</url>
- </snapshotRepository>
- </distributionManagement>
</project>
\ No newline at end of file
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/AbstractPatternDetectionProgram.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/AbstractPatternDetectionProgram.java
new file mode 100644
index 0000000..31932b1
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/AbstractPatternDetectionProgram.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.pattern.detection.flink;
+
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.streampipes.processors.pattern.detection.flink.config.PatternDetectionFlinkConfig;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public abstract class AbstractPatternDetectionProgram<B extends EventProcessorBindingParams> extends FlinkDataProcessorRuntime<B> {
+
+ public AbstractPatternDetectionProgram(B params, boolean debug) {
+ super(params, debug);
+ }
+
+ public AbstractPatternDetectionProgram(B params) {
+ super(params, false);
+ }
+
+ @Override
+ protected FlinkDeploymentConfig getDeploymentConfig() {
+ return new FlinkDeploymentConfig(PatternDetectionFlinkConfig.JAR_FILE,
+ PatternDetectionFlinkConfig.INSTANCE.getFlinkHost(), PatternDetectionFlinkConfig.INSTANCE.getFlinkPort());
+ }
+
+ @Override
+ public void appendEnvironmentConfig(StreamExecutionEnvironment env) {
+ env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
+ env.setParallelism(1);
+ }
+
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/PatternDetectionFlinkInit.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/PatternDetectionFlinkInit.java
index 326f8c8..92bd858 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/PatternDetectionFlinkInit.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/PatternDetectionFlinkInit.java
@@ -16,20 +16,21 @@
package org.streampipes.processors.pattern.detection.flink;
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
import org.streampipes.processors.pattern.detection.flink.config.PatternDetectionFlinkConfig;
import org.streampipes.processors.pattern.detection.flink.processor.increase.IncreaseController;
import org.streampipes.processors.pattern.detection.flink.processor.peak.PeakDetectionController;
-import org.streampipes.processors.pattern.detection.flink.processor.sequence.SequenceController;
-import org.streampipes.container.init.DeclarersSingleton;
-import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
public class PatternDetectionFlinkInit extends StandaloneModelSubmitter {
public static void main(String[] args) {
DeclarersSingleton.getInstance()
.add(new IncreaseController())
- .add(new PeakDetectionController())
- .add(new SequenceController());
+ .add(new PeakDetectionController());
+ //.add(new SequenceController())
+ //.add(new AbsenceController())
+ //.add(new AndController());
new PatternDetectionFlinkInit().init(PatternDetectionFlinkConfig.INSTANCE);
}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
index 4a9e89f..82629f9 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/config/PatternDetectionFlinkConfig.java
@@ -23,9 +23,9 @@
INSTANCE;
private SpConfig config;
- public static final String JAR_FILE = "./streampipes-examples-flink.jar";
+ public static final String JAR_FILE = "./streampipes-processing-element-container.jar";
- private final static String service_id = "pe/org.streampipes.processors.pattern.detection.flink.jvm";
+ private final static String service_id = "pe/org.streampipes.processors.pattern.detection.flink";
private final static String service_name = "Processors Pattern Detection Flink";
private final static String service_container_name = "processors-pattern-detection-flink";
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/Absence.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/Absence.java
new file mode 100644
index 0000000..64688ab
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/Absence.java
@@ -0,0 +1,19 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.pattern.detection.flink.processor.absence;
+
+public class Absence {
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceController.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceController.java
new file mode 100644
index 0000000..7b76c49
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceController.java
@@ -0,0 +1,58 @@
+package org.streampipes.processors.pattern.detection.flink.processor.absence;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.EventProperty;
+import org.streampipes.processors.pattern.detection.flink.config.PatternDetectionFlinkConfig;
+import org.streampipes.processors.pattern.detection.flink.processor.and.TimeUnit;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AbsenceController extends FlinkDataProcessorDeclarer<AbsenceParameters> {
+
+ private static final String TIME_WINDOW = "time-window";
+ private static final String TIME_UNIT = "time-unit";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.pattern-detection.flink.absence", "Absence", "Detects whether an event does not arrive within a specified time after the occurrence of another event.")
+ .category(DataProcessorType.PATTERN_DETECT)
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredProperty(EpRequirements.anyProperty())
+ .build())
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredProperty(EpRequirements.anyProperty())
+ .build())
+ .requiredSingleValueSelection(Labels.from(TIME_UNIT, "Time Unit", "The time unit used for detecting the co-occurrence."), Options.from("Seconds", "Minutes", "Hours"))
+ .requiredIntegerParameter(Labels.from(TIME_WINDOW, "Time Window Size", "Time window size (seconds)"))
+ .outputStrategy(OutputStrategies.custom(false))
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<AbsenceParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ List<String> selectProperties = new ArrayList<>();
+ for (EventProperty p : graph.getOutputStream().getEventSchema().getEventProperties()) {
+ selectProperties.add(p.getRuntimeName());
+ }
+
+ TimeUnit timeUnit = TimeUnit.valueOf(extractor.selectedSingleValue(TIME_UNIT, String.class));
+ Integer timeWindow = extractor.singleValueParameter(TIME_WINDOW, Integer.class);
+
+ AbsenceParameters params = new AbsenceParameters(graph, selectProperties, timeWindow, timeUnit);
+
+ return new AbsenceProgram(params, PatternDetectionFlinkConfig.INSTANCE.getDebug());
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceParameters.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceParameters.java
new file mode 100644
index 0000000..1efcbca
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceParameters.java
@@ -0,0 +1,36 @@
+package org.streampipes.processors.pattern.detection.flink.processor.absence;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.pattern.detection.flink.processor.and.TimeUnit;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AbsenceParameters extends EventProcessorBindingParams {
+
+ private static final long serialVersionUID = 4319341875274736697L;
+
+ private List<String> selectProperties = new ArrayList<>();
+ private Integer timeWindowSize;
+ private TimeUnit timeUnit;
+
+ public AbsenceParameters(DataProcessorInvocation graph, List<String> selectProperties, Integer timeWindowSize, TimeUnit timeUnit) {
+ super(graph);
+ this.selectProperties = selectProperties;
+ this.timeWindowSize = timeWindowSize;
+ this.timeUnit = timeUnit;
+ }
+
+ public List<String> getSelectProperties() {
+ return selectProperties;
+ }
+
+ public Integer getTimeWindowSize() {
+ return timeWindowSize;
+ }
+
+ public TimeUnit getTimeUnit() {
+ return timeUnit;
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceProgram.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceProgram.java
new file mode 100644
index 0000000..0794647
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/absence/AbsenceProgram.java
@@ -0,0 +1,113 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.pattern.detection.flink.processor.absence;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.cep.CEP;
+import org.apache.flink.cep.PatternFlatSelectFunction;
+import org.apache.flink.cep.PatternFlatTimeoutFunction;
+import org.apache.flink.cep.PatternStream;
+import org.apache.flink.cep.pattern.Pattern;
+import org.apache.flink.cep.pattern.conditions.SimpleCondition;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.apache.flink.util.Collector;
+import org.apache.flink.util.OutputTag;
+import org.streampipes.processors.pattern.detection.flink.AbstractPatternDetectionProgram;
+import org.streampipes.processors.pattern.detection.flink.processor.and.TimeUnitConverter;
+
+import java.util.List;
+import java.util.Map;
+
+public class AbsenceProgram extends AbstractPatternDetectionProgram<AbsenceParameters> {
+
+ public AbsenceProgram(AbsenceParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ @Override
+ public DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... messageStream) {
+
+ Time time = TimeUnitConverter.toTime(params.getTimeUnit(), params.getTimeWindowSize());
+
+ DataStream<Tuple2<Boolean, Map<String, Object>>> stream1 = messageStream[0].flatMap(new FlatMapFunction<Map<String, Object>, Tuple2<Boolean, Map<String, Object>>>() {
+ @Override
+ public void flatMap(Map<String, Object> in, Collector<Tuple2<Boolean, Map<String, Object>>> out) throws Exception {
+ out.collect(new Tuple2<>(true, in));
+ }
+ });
+
+ DataStream<Tuple2<Boolean, Map<String, Object>>> stream2 = messageStream[1].flatMap(new FlatMapFunction<Map<String, Object>, Tuple2<Boolean, Map<String, Object>>>() {
+ @Override
+ public void flatMap(Map<String, Object> in, Collector<Tuple2<Boolean, Map<String, Object>>> out) throws Exception {
+ out.collect(new Tuple2<>(false, in));
+ }
+ });
+
+ DataStream<Tuple2<Boolean, Map<String, Object>>> joinedStreams = stream2.union(stream1);
+
+
+ Pattern<Tuple2<Boolean, Map<String, Object>>, Tuple2<Boolean, Map<String, Object>>> matchedEvents =
+ Pattern.<Tuple2<Boolean, Map<String, Object>>>begin("start")
+ .where(new SimpleCondition<Tuple2<Boolean, Map<String, Object>>>() {
+ @Override
+ public boolean filter(Tuple2<Boolean, Map<String, Object>> ride) throws Exception {
+ return ride.f0;
+ }
+ })
+ .next("end")
+ .where(new SimpleCondition<Tuple2<Boolean, Map<String, Object>>>() {
+ @Override
+ public boolean filter(Tuple2<Boolean, Map<String, Object>> ride) throws Exception {
+ return !ride.f0;
+ }
+ });
+
+ PatternStream<Tuple2<Boolean, Map<String, Object>>> patternStream = CEP.pattern(joinedStreams, matchedEvents.within(time));
+
+ OutputTag<Tuple2<Boolean, Map<String, Object>>> timedout = new OutputTag<Tuple2<Boolean, Map<String, Object>>>("timedout"){};
+
+ SingleOutputStreamOperator<Tuple2<Boolean, Map<String, Object>>> matched = patternStream.flatSelect(
+ timedout,
+ new TimedOut(),
+ new FlatSelectNothing<>()
+ );
+
+ return matched.getSideOutput(timedout).flatMap(new FlatMapFunction<Tuple2<Boolean, Map<String, Object>>, Map<String, Object>>() {
+ @Override
+ public void flatMap(Tuple2<Boolean, Map<String, Object>> in, Collector<Map<String, Object>> out) throws Exception {
+ out.collect(in.f1);
+ }
+ });
+ }
+
+ public static class TimedOut implements PatternFlatTimeoutFunction<Tuple2<Boolean, Map<String, Object>>, Tuple2<Boolean, Map<String, Object>>> {
+ @Override
+ public void timeout(Map<String, List<Tuple2<Boolean, Map<String, Object>>>> map, long l, Collector<Tuple2<Boolean, Map<String, Object>>> collector) throws Exception {
+ Tuple2<Boolean, Map<String, Object>> rideStarted = map.get("start").get(0);
+ collector.collect(rideStarted);
+ }
+ }
+
+ public static class FlatSelectNothing<T> implements PatternFlatSelectFunction<T, T> {
+ @Override
+ public void flatSelect(Map<String, List<T>> pattern, Collector<T> collector) {
+
+ }
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/And.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/And.java
new file mode 100644
index 0000000..9d66dab
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/And.java
@@ -0,0 +1,19 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.pattern.detection.flink.processor.and;
+
+public class And {
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndController.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndController.java
new file mode 100644
index 0000000..53037f2
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndController.java
@@ -0,0 +1,57 @@
+package org.streampipes.processors.pattern.detection.flink.processor.and;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.pattern.detection.flink.config.PatternDetectionFlinkConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+import java.util.List;
+
+public class AndController extends FlinkDataProcessorDeclarer<AndParameters> {
+
+ private static final String TIME_WINDOW = "time-window";
+ private static final String TIME_UNIT = "time-unit";
+ private static final String LEFT_MAPPING = "left-mapping";
+ private static final String RIGHT_MAPPING = "right-mapping";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.pattern-detection.flink.and", "And", "Detects whether an event co-occurs with another event within a given time..")
+ .category(DataProcessorType.PATTERN_DETECT)
+ .iconUrl(PatternDetectionFlinkConfig.getIconUrl("And_Icon"))
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithNaryMapping(EpRequirements.anyProperty(), Labels.from("left-mapping", "Left Mapping", ""), PropertyScope.DIMENSION_PROPERTY)
+ .build())
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithNaryMapping(EpRequirements.anyProperty(), Labels.from("right-mapping", "Right Mapping", ""), PropertyScope.DIMENSION_PROPERTY)
+ .build())
+ .requiredSingleValueSelection(Labels.from(TIME_UNIT, "Time Unit", "The time unit used for detecting the co-occurrence."), Options.from("Seconds", "Minutes", "Hours"))
+ .requiredIntegerParameter(Labels.from(TIME_WINDOW, "Time Window Size", "Time window size (seconds)"))
+ .outputStrategy(OutputStrategies.custom(true))
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .build();
+ }
+
+
+ @Override
+ public FlinkDataProcessorRuntime<AndParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ List<String> leftMappings = extractor.mappingPropertyValues(LEFT_MAPPING);
+ List<String> rightMappings = extractor.mappingPropertyValues(RIGHT_MAPPING);
+ TimeUnit timeUnit = TimeUnit.valueOf(extractor.selectedSingleValue(TIME_UNIT, String.class));
+ Integer timeWindow = extractor.singleValueParameter(TIME_WINDOW, Integer.class);
+
+ AndParameters params = new AndParameters(graph, timeUnit, timeWindow, leftMappings, rightMappings);
+ return new AndProgram(params, PatternDetectionFlinkConfig.INSTANCE.getDebug());
+
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndParameters.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndParameters.java
new file mode 100644
index 0000000..a20ae8b
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndParameters.java
@@ -0,0 +1,40 @@
+package org.streampipes.processors.pattern.detection.flink.processor.and;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+import java.util.List;
+
+public class AndParameters extends EventProcessorBindingParams {
+
+ private TimeUnit timeUnit;
+ private Integer timeWindow;
+
+ private List<String> leftMappings;
+ private List<String> rightMappings;
+
+
+ public AndParameters(DataProcessorInvocation invocationGraph, TimeUnit timeUnit, Integer timeWindow, List<String> leftMappings, List<String> rightMappings) {
+ super(invocationGraph);
+ this.timeUnit = timeUnit;
+ this.timeWindow = timeWindow;
+ this.leftMappings = leftMappings;
+ this.rightMappings = rightMappings;
+ }
+
+ public TimeUnit getTimeUnit() {
+ return timeUnit;
+ }
+
+ public Integer getTimeWindow() {
+ return timeWindow;
+ }
+
+ public List<String> getLeftMappings() {
+ return leftMappings;
+ }
+
+ public List<String> getRightMappings() {
+ return rightMappings;
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndProgram.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndProgram.java
new file mode 100644
index 0000000..c23a030
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/AndProgram.java
@@ -0,0 +1,73 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.pattern.detection.flink.processor.and;
+
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.time.Time;
+import org.streampipes.processors.pattern.detection.flink.AbstractPatternDetectionProgram;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+public class AndProgram extends AbstractPatternDetectionProgram<AndParameters> {
+
+ public AndProgram(AndParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ @Override
+ public DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... messageStream) {
+ // A AND B within x minutes
+ List<String> leftMappings = params.getLeftMappings();
+ List<String> rightMappings = params.getRightMappings();
+ Time time = TimeUnitConverter.toTime(params.getTimeUnit(), params.getTimeWindow());
+
+ return messageStream[0].join(messageStream[1])
+ .where(new KeySelector<Map<String,Object>, String>() {
+ @Override
+ public String getKey(Map<String, Object> stringObjectMap) throws Exception {
+ StringBuilder builder = new StringBuilder();
+ for (String key : leftMappings) {
+ builder.append(key);
+ }
+ return builder.toString();
+ }
+ }).equalTo(new KeySelector<Map<String,Object>, String>() {
+ @Override
+ public String getKey(Map<String, Object> stringObjectMap) throws Exception {
+ StringBuilder builder = new StringBuilder();
+ for (String key : rightMappings) {
+ builder.append(key);
+ }
+ return builder.toString();
+ }
+ }).window(TumblingEventTimeWindows.of(time))
+ .apply(new JoinFunction<Map<String,Object>, Map<String,Object>, Map<String, Object>>() {
+ @Override
+ public Map<String, Object> join(Map<String, Object> e1, Map<String, Object> e2) throws Exception {
+ Map<String, Object> map = new HashMap<>();
+ map.putAll(e1);
+ map.putAll(e2);
+ return map;
+ }
+ });
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/TimeUnit.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/TimeUnit.java
new file mode 100644
index 0000000..112d52d
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/TimeUnit.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.pattern.detection.flink.processor.and;
+
+public enum TimeUnit {
+ Seconds, Minutes, Hours;
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/TimeUnitConverter.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/TimeUnitConverter.java
new file mode 100644
index 0000000..3609aab
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/and/TimeUnitConverter.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.pattern.detection.flink.processor.and;
+
+import org.apache.flink.streaming.api.windowing.time.Time;
+
+public class TimeUnitConverter {
+
+ public static Time toTime(TimeUnit timeUnit, long value) {
+ if (timeUnit == TimeUnit.Seconds) {
+ return Time.seconds(value);
+ } else if (timeUnit == TimeUnit.Minutes) {
+ return Time.minutes(value);
+ } else {
+ return Time.hours(value);
+ }
+
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/common/TimestampExtractor.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/common/TimestampExtractor.java
new file mode 100644
index 0000000..3cb5e8d
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/common/TimestampExtractor.java
@@ -0,0 +1,34 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.pattern.detection.flink.processor.common;
+
+import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExtractor;
+
+import java.util.Map;
+
+public class TimestampExtractor extends AscendingTimestampExtractor<Map<String, Object>> {
+
+ private String timestampField;
+
+ public TimestampExtractor(String timestampField) {
+ this.timestampField = timestampField;
+ }
+
+ @Override
+ public long extractAscendingTimestamp(Map<String, Object> in) {
+ return Long.parseLong(String.valueOf(in.get(timestampField)));
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Increase.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Increase.java
index af47b7a..a8be6a7 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Increase.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Increase.java
@@ -57,11 +57,12 @@
}
if (values.size() > 0) {
if (operation == Operation.INCREASE) {
- if (values.get(values.size()-1) > values.get(0) * (1 + increaseValue / 100)) {
+ if (values.get(values.size()-1) >= values.get(0) * (1 + increaseValue / 100)) {
buildOutput(out, lastEvent);
}
} else {
- if (values.get(values.size()-1) > values.get(0) * (1 - increaseValue / 100)) {
+ // 1 <= 2 - (2* (1- 10 / 100) <=> 1 <= 2*0
+ if (values.get(values.size()-1) <= values.get(0) - (values.get(values.size()-1) * (1 - (increaseValue / 100)))) {
buildOutput(out, lastEvent);
}
}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseController.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseController.java
index 300f9c5..e1f8ffd 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseController.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseController.java
@@ -27,23 +27,26 @@
import org.streampipes.sdk.helpers.*;
import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
public class IncreaseController extends FlinkDataProcessorDeclarer<IncreaseParameters> {
private static final String PARTITION_BY = "partition-by";
private static final String TIMESTAMP = "timestamp";
+ private static final String VALUE_MAPPING = "value-mapping";
+ private static final String INCREASE = "increase";
+ private static final String DURATION = "duration";
+ private static final String OPERATION = "operation";
@Override
public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("increase", "Increase", "Detects the increase of a numerical field over a customizable time window. Example: A temperature value increases by 10 percent within 5 minutes.")
+ return ProcessingElementBuilder.create("org.streampipes.processors.pattern-detection.flink.increase", "Increase", "Detects the increase of a numerical field over a customizable time window. Example: A temperature value increases by 10 percent within 5 minutes.")
.category(DataProcessorType.PATTERN_DETECT)
.iconUrl(PatternDetectionFlinkConfig.getIconUrl("increase-icon"))
.requiredStream(StreamRequirementsBuilder
.create()
.requiredPropertyWithUnaryMapping(EpRequirements
- .numberReq(), Labels.from("mapping", "Value to observe", "Specifies the value that should be " +
- "monitored."), PropertyScope.MEASUREMENT_PROPERTY)
+ .numberReq(), Labels.from(VALUE_MAPPING, "Value to observe", "Specifies the value that should be " +
+ "monitored."), PropertyScope.MEASUREMENT_PROPERTY)
.requiredPropertyWithUnaryMapping(EpRequirements
.timestampReq(), Labels.from(TIMESTAMP, "Timestamp field", "The field that contains " +
"the event's timestamp"), PropertyScope.HEADER_PROPERTY)
@@ -51,13 +54,13 @@
Labels.from(PARTITION_BY, "Group by", "Partition the stream by a given id"), PropertyScope
.DIMENSION_PROPERTY)
.build())
- .requiredIntegerParameter("increase", "Percentage of Increase/Decrease", "Specifies the increase in " +
+ .requiredIntegerParameter(INCREASE, "Percentage of Increase/Decrease", "Specifies the increase in " +
"percent (e.g., 100 indicates an increase by 100 percent within the specified time window.", 0, 500, 1)
- .requiredIntegerParameter("duration", "Time Window Length (Seconds)", "Specifies the size of the time window in seconds.")
- .requiredSingleValueSelection("operation", "Increase/Decrease", "Specifies the type of operation the " +
- "processor should perform.", Options.from("Increase", "Decrease"))
+ .requiredIntegerParameter(Labels.from(DURATION, "Time Window Length (Seconds)", "Specifies the size of the time window in seconds."))
+ .requiredSingleValueSelection(Labels.from(OPERATION, "Increase/Decrease", "Specifies the type of operation the " +
+ "processor should perform."), Options.from("Increase", "Decrease"))
.outputStrategy(OutputStrategies.custom(true))
- .supportedProtocols(SupportedProtocols.kafka())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
.supportedFormats(SupportedFormats.jsonFormat())
.build();
}
@@ -66,28 +69,25 @@
public FlinkDataProcessorRuntime<IncreaseParameters> getRuntime(DataProcessorInvocation graph,
ProcessingElementParameterExtractor extractor) {
- String operation = extractor.selectedSingleValue("operation", String.class);
- Integer increase = extractor.singleValueParameter("increase", Integer.class);
- Integer duration = extractor.singleValueParameter("duration", Integer.class);
- String mapping = extractor.mappingPropertyValue("mapping");
+ String operation = extractor.selectedSingleValue(OPERATION, String.class);
+ Integer increase = extractor.singleValueParameter(INCREASE, Integer.class);
+ Integer duration = extractor.singleValueParameter(DURATION, Integer.class);
+ String mapping = extractor.mappingPropertyValue(VALUE_MAPPING);
String groupBy = extractor.mappingPropertyValue(PARTITION_BY);
String timestampField = extractor.mappingPropertyValue(TIMESTAMP);
IncreaseParameters params = new IncreaseParameters(graph, getOperation(operation), increase, duration, mapping,
groupBy, timestampField);
-
- if (PatternDetectionFlinkConfig.INSTANCE.getDebug()) {
- return new IncreaseProgram(params);
- } else {
- return new IncreaseProgram(params, new FlinkDeploymentConfig(PatternDetectionFlinkConfig.JAR_FILE,
- PatternDetectionFlinkConfig.INSTANCE.getFlinkHost(), PatternDetectionFlinkConfig.INSTANCE.getFlinkPort()));
- }
+ return new IncreaseProgram(params, PatternDetectionFlinkConfig.INSTANCE.getDebug());
}
private Operation getOperation(String operation) {
- if (operation.equals("Increase")) return Operation.INCREASE;
- else return Operation.DECREASE;
+ if (operation.equals("Increase")) {
+ return Operation.INCREASE;
+ } else {
+ return Operation.DECREASE;
+ }
}
}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseProgram.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseProgram.java
index 5fd04d2..b23761c 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseProgram.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/IncreaseProgram.java
@@ -17,34 +17,29 @@
package org.streampipes.processors.pattern.detection.flink.processor.increase;
import org.apache.flink.api.java.functions.KeySelector;
-import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows;
+import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
import org.apache.flink.streaming.api.windowing.time.Time;
-import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.processors.pattern.detection.flink.AbstractPatternDetectionProgram;
+import org.streampipes.processors.pattern.detection.flink.processor.common.TimestampExtractor;
import java.util.Map;
-public class IncreaseProgram extends FlinkDataProcessorRuntime<IncreaseParameters> {
+public class IncreaseProgram extends AbstractPatternDetectionProgram<IncreaseParameters> {
- public IncreaseProgram(IncreaseParameters params) {
- super(params);
- setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
- }
-
- public IncreaseProgram(IncreaseParameters params, FlinkDeploymentConfig config) {
- super(params, config);
- setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
+ public IncreaseProgram(IncreaseParameters params, boolean debug) {
+ super(params, debug);
}
@Override
- protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... dataStreams) {
+ public DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... dataStreams) {
+ String timestampField = params.getTimestampField();
return dataStreams[0]
+ .assignTimestampsAndWatermarks(new TimestampExtractor(timestampField))
.keyBy(getKeySelector())
- .window(SlidingEventTimeWindows.of(Time.seconds(params.getDuration()), Time.seconds(1)))
+ .window(TumblingEventTimeWindows.of(Time.seconds(params.getDuration())))
.apply(new Increase(params.getIncrease(), params.getOperation(), params.getMapping(), params
- .getOutputProperties(), params.getGroupBy()));
+ .getOutputProperties(), params.getGroupBy())).setParallelism(1);
}
private KeySelector<Map<String, Object>, String> getKeySelector() {
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Operation.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Operation.java
index 1b8e04d..c667f79 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Operation.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/increase/Operation.java
@@ -17,5 +17,5 @@
package org.streampipes.processors.pattern.detection.flink.processor.increase;
public enum Operation {
-INCREASE, DECREASE;
+INCREASE, DECREASE
}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionController.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionController.java
index 337b1ab..f1d2168 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionController.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionController.java
@@ -27,7 +27,6 @@
import org.streampipes.sdk.helpers.*;
import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
/**
* Created by riemer on 20.04.2017.
@@ -44,7 +43,7 @@
@Override
public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("peak-detection", "Peak Detection",
+ return ProcessingElementBuilder.create("org.streampipes.processors.pattern-detection.flink.peak-detection", "Peak Detection",
"Detect peaks in time series data")
.category(DataProcessorType.ALGORITHM)
.iconUrl(PatternDetectionFlinkConfig.getIconUrl("peak-detection-icon"))
@@ -57,19 +56,19 @@
.requiredPropertyWithUnaryMapping(EpRequirements.stringReq(),
Labels.from(PARTITION_BY, "Group by", "Partition the stream by a given id"), PropertyScope
.DIMENSION_PROPERTY).build())
- .requiredIntegerParameter(COUNT_WINDOW_SIZE, "Count Window Size", "Defines " +
- "the size of the count window", 60)
- .requiredIntegerParameter(LAG_KEY, "Lag", "Defines the lag of the smoothing " +
- "function", 5)
- .requiredFloatParameter(THRESHOLD_KEY, "Threshold", "Defines the standard deviation " +
- "threshold", 2.0f)
- .requiredFloatParameter(INFLUENCE_KEY, "Influence", "Defines the influence", 0.5f)
+ .requiredIntegerParameter(Labels.from(COUNT_WINDOW_SIZE, "Count Window Size", "Defines " +
+ "the size of the count window"), 60)
+ .requiredIntegerParameter(Labels.from(LAG_KEY, "Lag", "Defines the lag of the smoothing " +
+ "function"), 5)
+ .requiredFloatParameter(Labels.from(THRESHOLD_KEY, "Threshold", "Defines the standard deviation " +
+ "threshold"), 2.0f)
+ .requiredFloatParameter(Labels.from(INFLUENCE_KEY, "Influence", "Defines the influence"), 0.5f)
.outputStrategy(OutputStrategies.fixed(
EpProperties.timestampProperty("timestamp"),
EpProperties.stringEp(Labels.empty(), "id", "http://schema.org/id"),
EpProperties.integerEp(Labels.empty(), "signal", "http://schema.org/Number")))
.supportedFormats(SupportedFormats.jsonFormat())
- .supportedProtocols(SupportedProtocols.kafka())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
.build();
}
@@ -90,13 +89,6 @@
PeakDetectionParameters params = new PeakDetectionParameters(sepa,
valueToObserve, timestampMapping, groupBy, countWindowSize, lag, threshold, influence);
- if (PatternDetectionFlinkConfig.INSTANCE.getDebug()) {
- return new PeakDetectionProgram(params);
- } else {
- return new PeakDetectionProgram(params, new FlinkDeploymentConfig(PatternDetectionFlinkConfig.JAR_FILE,
- PatternDetectionFlinkConfig.INSTANCE.getFlinkHost(), PatternDetectionFlinkConfig.INSTANCE.getFlinkPort()));
- }
-
-
+ return new PeakDetectionProgram(params, PatternDetectionFlinkConfig.INSTANCE.getDebug());
}
}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionProgram.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionProgram.java
index c519b90..e6b89db 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionProgram.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/peak/PeakDetectionProgram.java
@@ -20,9 +20,8 @@
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processors.pattern.detection.flink.AbstractPatternDetectionProgram;
import org.streampipes.processors.pattern.detection.flink.processor.peak.utils.SlidingBatchWindow;
-import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
import java.util.List;
import java.util.Map;
@@ -30,15 +29,10 @@
/**
* Created by riemer on 20.04.2017.
*/
-public class PeakDetectionProgram extends FlinkDataProcessorRuntime<PeakDetectionParameters> {
+public class PeakDetectionProgram extends AbstractPatternDetectionProgram<PeakDetectionParameters> {
- public PeakDetectionProgram(PeakDetectionParameters params) {
- super(params);
- }
-
- public PeakDetectionProgram(PeakDetectionParameters params,
- FlinkDeploymentConfig config) {
- super(params, config);
+ public PeakDetectionProgram(PeakDetectionParameters params, boolean debug) {
+ super(params, debug);
}
@Override
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceController.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceController.java
index 2faaf78..d1f928c 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceController.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceController.java
@@ -26,7 +26,6 @@
import org.streampipes.sdk.helpers.*;
import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
public class SequenceController extends FlinkDataProcessorDeclarer<SequenceParameters> {
@@ -35,17 +34,17 @@
@Override
public DataProcessorDescription declareModel() {
- return ProcessingElementBuilder.create("sequence", "Sequence", "Detects a sequence of events in the following form: Event A followed by Event B within X seconds. In addition, both streams can be matched by a common property value (e.g., a.machineId = b.machineId)")
+ return ProcessingElementBuilder.create("org.streampipes.processors.pattern-detection.flink.sequence", "Sequence", "Detects a sequence of events in the following form: Event A followed by Event B within X seconds. In addition, both streams can be matched by a common property value (e.g., a.machineId = b.machineId)")
.category(DataProcessorType.PATTERN_DETECT)
.iconUrl(PatternDetectionFlinkConfig.getIconUrl("Sequence_Icon_HQ"))
.requiredStream(StreamRequirementsBuilder.create().requiredProperty(EpRequirements.anyProperty()).build())
.requiredStream(StreamRequirementsBuilder.create().requiredProperty(EpRequirements.anyProperty()).build())
- .requiredIntegerParameter(TIME_WINDOW, "Time Window Size", "Size of the time window ")
- .requiredSingleValueSelection(TIME_UNIT, "Time Unit", "Specifies a unit for the time window of the " +
- "sequence. ", Options.from("sec", "min", "hrs"))
+ .requiredIntegerParameter(Labels.from(TIME_WINDOW, "Time Window Size", "Size of the time window "))
+ .requiredSingleValueSelection(Labels.from(TIME_UNIT, "Time Unit", "Specifies a unit for the time window of the " +
+ "sequence. "), Options.from("sec", "min", "hrs"))
.outputStrategy(OutputStrategies.keep(false))
.supportedFormats(SupportedFormats.jsonFormat())
- .supportedProtocols(SupportedProtocols.kafka())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
.build();
}
@@ -58,12 +57,7 @@
SequenceParameters params = new SequenceParameters(graph, timeWindowSize, timeUnit);
- if (PatternDetectionFlinkConfig.INSTANCE.getDebug()) {
- return new SequenceProgram(params);
- } else {
- return new SequenceProgram(params, new FlinkDeploymentConfig(PatternDetectionFlinkConfig.JAR_FILE,
- PatternDetectionFlinkConfig.INSTANCE.getFlinkHost(), PatternDetectionFlinkConfig.INSTANCE.getFlinkPort()));
- }
+ return new SequenceProgram(params, PatternDetectionFlinkConfig.INSTANCE.getDebug());
}
}
diff --git a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceProgram.java b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceProgram.java
index f618d28..effee04 100644
--- a/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceProgram.java
+++ b/streampipes-processors-pattern-detection-flink/src/main/java/org/streampipes/processors/pattern/detection/flink/processor/sequence/SequenceProgram.java
@@ -18,19 +18,14 @@
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
-import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.processors.pattern.detection.flink.AbstractPatternDetectionProgram;
import java.util.Map;
-public class SequenceProgram extends FlinkDataProcessorRuntime<SequenceParameters> {
+public class SequenceProgram extends AbstractPatternDetectionProgram<SequenceParameters> {
- public SequenceProgram(SequenceParameters params) {
- super(params);
- }
-
- public SequenceProgram(SequenceParameters params, FlinkDeploymentConfig config) {
- super(params, config);
+ public SequenceProgram(SequenceParameters params, boolean debug) {
+ super(params, debug);
}
@Override
diff --git a/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/absence/TestAbsence.java b/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/absence/TestAbsence.java
new file mode 100644
index 0000000..1fecea7
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/absence/TestAbsence.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.pattern.detection.processor.absence;
+
+import io.flinkspector.datastream.DataStreamTestBase;
+import io.flinkspector.datastream.input.EventTimeInput;
+import io.flinkspector.datastream.input.EventTimeInputBuilder;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.streampipes.processors.pattern.detection.flink.processor.absence.AbsenceController;
+import org.streampipes.processors.pattern.detection.flink.processor.absence.AbsenceParameters;
+import org.streampipes.processors.pattern.detection.flink.processor.absence.AbsenceProgram;
+import org.streampipes.processors.pattern.detection.flink.processor.and.TimeUnit;
+import org.streampipes.test.generator.InvocationGraphGenerator;
+
+import java.util.*;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+
+@RunWith(Parameterized.class)
+public class TestAbsence extends DataStreamTestBase {
+
+ @Parameterized.Parameters
+ public static Iterable<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {10, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), true, 12},
+ {10, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), false, 5},
+ {5, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), true, 6},
+ });
+ }
+
+ @Parameterized.Parameter
+ public Integer timeWindow;
+
+ @Parameterized.Parameter(1)
+ public TimeUnit timeUnit;
+
+ @Parameterized.Parameter(2)
+ public List<String> leftMapping;
+
+ @Parameterized.Parameter(3)
+ public List<String> rightMapping;
+
+ @Parameterized.Parameter(4)
+ public Boolean shouldMatch;
+
+ @Parameterized.Parameter(5)
+ public Integer waitForMs;
+
+
+ @Test
+ public void testAbsenceProgram() {
+ AbsenceParameters params = new AbsenceParameters(InvocationGraphGenerator.makeEmptyInvocation(new AbsenceController().declareModel()), Arrays.asList("id", "timestamp", "value"), timeWindow, timeUnit);
+
+ AbsenceProgram program = new AbsenceProgram(params, true);
+
+ DataStream<Map<String, Object>> stream = program.getApplicationLogic(createTestStream(makeInputData(1, makeMap(), 0)), createTestStream(makeInputData(waitForMs, makeMap(), 1)));
+
+ assertStream(stream, equalTo(getOutput(shouldMatch)));
+ }
+
+ private Collection<Map<String, Object>> getOutput(Boolean shouldMatch) {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+
+ if (shouldMatch) {
+ allEvents.add(makeMap().get(0));
+ }
+
+ return allEvents;
+ }
+
+ private EventTimeInput<Map<String, Object>> makeInputData(Integer delayEvent, List<Map<String, Object>> inputMap, Integer i) {
+ List<Map<String, Object>> testData = inputMap;
+ EventTimeInputBuilder<Map<String, Object>> builder = EventTimeInputBuilder.startWith(testData.get(i), after(delayEvent, seconds));
+
+ return builder;
+ }
+
+ private List<Map<String, Object>> makeMap() {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+ Map<String, Object> event1 = new HashMap<>();
+ event1.put("id", "a");
+ event1.put("timestamp", 0);
+
+ allEvents.add(event1);
+
+ Map<String, Object> event2 = new HashMap<>();
+ event2.put("id", "a");
+ event2.put("timestamp", waitForMs);
+
+ allEvents.add(event2);
+
+ return allEvents;
+ }
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/and/TestAnd.java b/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/and/TestAnd.java
new file mode 100644
index 0000000..b98a5bb
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/and/TestAnd.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.pattern.detection.processor.and;
+
+import io.flinkspector.datastream.DataStreamTestBase;
+import io.flinkspector.datastream.input.EventTimeInput;
+import io.flinkspector.datastream.input.EventTimeInputBuilder;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.streampipes.processors.pattern.detection.flink.processor.and.AndController;
+import org.streampipes.processors.pattern.detection.flink.processor.and.AndParameters;
+import org.streampipes.processors.pattern.detection.flink.processor.and.AndProgram;
+import org.streampipes.processors.pattern.detection.flink.processor.and.TimeUnit;
+import org.streampipes.test.generator.InvocationGraphGenerator;
+
+import java.util.*;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+
+@RunWith(Parameterized.class)
+public class TestAnd extends DataStreamTestBase {
+
+
+ @Parameterized.Parameters
+ public static Iterable<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {2, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), true, 1, 1},
+ {1, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), true, 1, 1},
+ {10, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), false, 1, 12},
+ {10, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), true, 3, 4},
+ {1, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), false, 1, 2},
+ {3600, TimeUnit.Seconds, Arrays.asList("id"), Arrays.asList("id"), true, 10, 3500},
+
+ });
+ }
+
+ @Parameterized.Parameter
+ public Integer timeWindow;
+
+ @Parameterized.Parameter(1)
+ public TimeUnit timeUnit;
+
+ @Parameterized.Parameter(2)
+ public List<String> leftMapping;
+
+ @Parameterized.Parameter(3)
+ public List<String> rightMapping;
+
+ @Parameterized.Parameter(4)
+ public Boolean shouldMatch;
+
+ @Parameterized.Parameter(5)
+ public Integer delayFirstEvent;
+
+ @Parameterized.Parameter(6)
+ public Integer delaySecondEvent;
+
+ @Test
+ public void testAndProgram() {
+ AndParameters params = new AndParameters(InvocationGraphGenerator.makeEmptyInvocation(new AndController().declareModel()), timeUnit, timeWindow, leftMapping, rightMapping);
+
+ AndProgram program = new AndProgram(params, true);
+
+ DataStream<Map<String, Object>> stream = program.getApplicationLogic(createTestStream(makeInputData(delayFirstEvent, makeMap("field1"))), createTestStream(makeInputData(delaySecondEvent, makeMap("field2"))));
+
+ assertStream(stream, equalTo(getOutput(shouldMatch)));
+ }
+
+ private Collection<Map<String, Object>> getOutput(Boolean shouldMatch) {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+
+ if (shouldMatch) {
+ Map<String, Object> outMap = new HashMap<>();
+ outMap.put("id", "a");
+ outMap.put("field1", 1);
+ outMap.put("field2", 1);
+ allEvents.add(outMap);
+ }
+
+ return allEvents;
+ }
+
+ private EventTimeInput<Map<String, Object>> makeInputData(Integer delayEvent, List<Map<String, Object>> inputMap) {
+ List<Map<String, Object>> testData = inputMap;
+ EventTimeInputBuilder<Map<String, Object>> builder = EventTimeInputBuilder.startWith(testData.get(0), after(delayEvent, seconds));
+
+ return builder;
+ }
+
+ private List<Map<String, Object>> makeMap(String fieldName) {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+ Map<String, Object> event = new HashMap<>();
+ event.put("id", "a");
+ event.put(fieldName, 1);
+
+ allEvents.add(event);
+
+ return allEvents;
+ }
+
+}
diff --git a/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/increase/TestIncrease.java b/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/increase/TestIncrease.java
new file mode 100644
index 0000000..2cdec4d
--- /dev/null
+++ b/streampipes-processors-pattern-detection-flink/src/test/java/org/streampipes/processors/pattern/detection/processor/increase/TestIncrease.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.pattern.detection.processor.increase;
+
+import io.flinkspector.core.input.Input;
+import io.flinkspector.core.input.InputBuilder;
+import io.flinkspector.datastream.DataStreamTestBase;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.streampipes.processors.pattern.detection.flink.processor.increase.IncreaseController;
+import org.streampipes.processors.pattern.detection.flink.processor.increase.IncreaseParameters;
+import org.streampipes.processors.pattern.detection.flink.processor.increase.IncreaseProgram;
+import org.streampipes.processors.pattern.detection.flink.processor.increase.Operation;
+import org.streampipes.test.generator.InvocationGraphGenerator;
+
+import java.util.*;
+
+import static org.hamcrest.core.IsEqual.equalTo;
+
+@RunWith(Parameterized.class)
+public class TestIncrease extends DataStreamTestBase {
+
+ @Parameterized.Parameters
+ public static Iterable<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {Operation.INCREASE, 100, 10, 1.0f, 2.0f, 5000, true},
+ {Operation.DECREASE, 100, 10, 2.0f, 0.0f, 5000, true},
+ {Operation.DECREASE, 50, 10, 2.0f, 1.0f, 5000, true},
+ {Operation.DECREASE, 50, 10, 2.0f, 1.5f, 5000, false},
+ {Operation.INCREASE, 100, 10, 1.0f, 2.0f, 11000, false},
+ });
+ }
+
+ @Parameterized.Parameter
+ public Operation operation;
+
+ @Parameterized.Parameter(1)
+ public Integer increase;
+
+ @Parameterized.Parameter(2)
+ public Integer duration;
+
+ @Parameterized.Parameter(3)
+ public Float value1;
+
+ @Parameterized.Parameter(4)
+ public Float value2;
+
+ @Parameterized.Parameter(5)
+ public Integer waitForMs;
+
+ @Parameterized.Parameter(6)
+ public Boolean shouldMatch;
+
+
+ @Test
+ public void testIncreaseProgram() {
+ IncreaseParameters params = new IncreaseParameters(InvocationGraphGenerator
+ .makeInvocationWithOutputProperties(new IncreaseController().declareModel(), Arrays.asList("id", "timestamp", "value")),
+ operation,
+ increase,
+ duration,
+ "value",
+ "id",
+ "timestamp");
+
+
+ IncreaseProgram program = new IncreaseProgram(params, true);
+
+ DataStream<Map<String, Object>> stream = program.getApplicationLogic(createTestStream(makeInputData(makeMap())));
+
+ assertStream(stream, equalTo(getOutput(shouldMatch)));
+ }
+
+ private Collection<Map<String, Object>> getOutput(Boolean shouldMatch) {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+
+ if (shouldMatch) {
+ allEvents.add(makeMap().get(1));
+ }
+
+ return allEvents;
+ }
+
+ private Input<Map<String, Object>> makeInputData(List<Map<String, Object>> inputMap) {
+ List<Map<String, Object>> testData = inputMap;
+ InputBuilder<Map<String, Object>> builder = InputBuilder.startWith(testData.get(0));
+ for(int i = 1; i < inputMap.size(); i++) {
+ builder.emit(inputMap.get(i));
+ }
+ return builder;
+ }
+
+ private List<Map<String, Object>> makeMap() {
+ List<Map<String, Object>> allEvents = new ArrayList<>();
+ Map<String, Object> event1 = new HashMap<>();
+ event1.put("id", "a");
+ event1.put("timestamp", 0);
+ event1.put("value", value1);
+
+ allEvents.add(event1);
+
+ Map<String, Object> event2 = new HashMap<>();
+ event2.put("id", "a");
+ event2.put("timestamp", waitForMs);
+ event2.put("value", value2);
+
+ allEvents.add(event2);
+
+ return allEvents;
+ }
+
+
+}
diff --git a/streampipes-processors-statistics-flink/Dockerfile b/streampipes-processors-statistics-flink/Dockerfile
new file mode 100644
index 0000000..2a636ee
--- /dev/null
+++ b/streampipes-processors-statistics-flink/Dockerfile
@@ -0,0 +1,8 @@
+FROM anapsix/alpine-java
+
+EXPOSE 8090
+ENV CONSUL_LOCATION consul
+
+ADD ./target/streampipes-processors-statistics-flink.jar /streampipes-processing-element-container.jar
+
+ENTRYPOINT ["java", "-jar", "/streampipes-processing-element-container.jar"]
diff --git a/streampipes-processors-statistics-flink/deployment/docker-compose.yml b/streampipes-processors-statistics-flink/deployment/docker-compose.yml
new file mode 100644
index 0000000..a67b91d
--- /dev/null
+++ b/streampipes-processors-statistics-flink/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-statistics-flink:
+ image: ${SP_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-statistics-flink:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-statistics-flink/development/.env b/streampipes-processors-statistics-flink/development/.env
new file mode 100644
index 0000000..cdf33dc
--- /dev/null
+++ b/streampipes-processors-statistics-flink/development/.env
@@ -0,0 +1,8 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6045
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_KAFKA_HOST=localhost
+SP_ZOOKEEPER_HOST=localhost
+SP_FLINK_DEBUG=true
+SP_ELASTICSEARCH_HOST=localhost
diff --git a/streampipes-processors-statistics-flink/pom.xml b/streampipes-processors-statistics-flink/pom.xml
new file mode 100644
index 0000000..de1511e
--- /dev/null
+++ b/streampipes-processors-statistics-flink/pom.xml
@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-processors-statistics-flink</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-commons</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-flink</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.9.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ <version>3.6.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.8.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.24</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.streampipes.processors.statistics.flink.StatisticsFlinkInit</mainClass>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
+ </exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>streampipes-processors-statistics-flink</finalName>
+
+ </build>
+</project>
\ No newline at end of file
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/AbstractStatisticsProgram.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/AbstractStatisticsProgram.java
new file mode 100644
index 0000000..ce4ffe5
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/AbstractStatisticsProgram.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.statistics.flink;
+
+import org.streampipes.processors.statistics.flink.config.StatisticsFlinkConfig;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public abstract class AbstractStatisticsProgram<B extends EventProcessorBindingParams> extends FlinkDataProcessorRuntime<B> {
+
+ public AbstractStatisticsProgram(B params, boolean debug) {
+ super(params, debug);
+ }
+
+ public AbstractStatisticsProgram(B params) {
+ super(params, false);
+ }
+
+ @Override
+ protected FlinkDeploymentConfig getDeploymentConfig() {
+ return new FlinkDeploymentConfig(StatisticsFlinkConfig.JAR_FILE,
+ StatisticsFlinkConfig.INSTANCE.getFlinkHost(), StatisticsFlinkConfig.INSTANCE.getFlinkPort());
+ }
+
+}
+
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/StatisticsFlinkInit.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/StatisticsFlinkInit.java
new file mode 100644
index 0000000..1f02a5d
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/StatisticsFlinkInit.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink;
+
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.processors.statistics.flink.config.StatisticsFlinkConfig;
+import org.streampipes.processors.statistics.flink.processor.stat.StatisticsSummaryController;
+import org.streampipes.processors.statistics.flink.processor.stat.window.StatisticsSummaryControllerWindow;
+
+public class StatisticsFlinkInit extends StandaloneModelSubmitter {
+
+ public static void main(String[] args) {
+ DeclarersSingleton.getInstance()
+ .add(new StatisticsSummaryController())
+ .add(new StatisticsSummaryControllerWindow());
+
+ new StatisticsFlinkInit().init(StatisticsFlinkConfig.INSTANCE);
+ }
+
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/config/ConfigKeys.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/config/ConfigKeys.java
new file mode 100644
index 0000000..b75ae02
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/config/ConfigKeys.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.config;
+
+public class ConfigKeys {
+ final static String HOST = "SP_HOST";
+ final static String PORT = "SP_PORT";
+ final static String FLINK_HOST = "SP_FLINK_HOST";
+ final static String FLINK_PORT = "SP_FLINK_PORT";
+ final static String ELASTIC_HOST = "SP_ELASTICSEARCH_HOST";
+ final static String ELASTIC_PORT = "SP_ELASTICSEARCH_PORT";
+ final static String ELASTIC_PORT_REST = "SP_ELASTICSEARCH_PORT_REST";
+ final static String ICON_HOST = "SP_ICON_HOST";
+ final static String ICON_PORT = "SP_ICON_PORT";
+ final static String SERVICE_NAME = "SP_SERVICE_NAME";
+ final static String DEBUG = "SP_FLINK_DEBUG";
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
new file mode 100644
index 0000000..48c940d
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/config/StatisticsFlinkConfig.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.config;
+
+import org.streampipes.config.SpConfig;
+import org.streampipes.container.model.PeConfig;
+
+public enum StatisticsFlinkConfig implements PeConfig {
+ INSTANCE;
+
+ private SpConfig config;
+ public static final String JAR_FILE = "./streampipes-processing-element-container.jar";
+
+
+ private final static String service_id = "pe/org.streampipes.processors.statistics.flink";
+ private final static String service_name = "Statistics Processors Flink";
+ private final static String service_container_name = "processors-statistics-flink";
+
+
+ StatisticsFlinkConfig() {
+ config = SpConfig.getSpConfig(service_id);
+
+ config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
+ config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
+ config.register(ConfigKeys.FLINK_HOST, "jobmanager", "Host for the flink cluster");
+ config.register(ConfigKeys.FLINK_PORT, 6123, "Port for the flink cluster");
+ config.register(ConfigKeys.ELASTIC_HOST, "elasticsearch", "Elastic search host address");
+ config.register(ConfigKeys.ELASTIC_PORT, 9300, "Elasitc search port");
+ config.register(ConfigKeys.ELASTIC_PORT_REST, 9200, "Elasitc search rest port");
+
+ config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host");
+ config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx");
+
+ config.register(ConfigKeys.DEBUG, false, "When set to true programs are not deployed to cluster, but executed locally");
+
+ config.register(ConfigKeys.SERVICE_NAME, service_name, "The name of the service");
+
+ }
+
+ @Override
+ public String getHost() {
+ return config.getString(ConfigKeys.HOST);
+ }
+
+ @Override
+ public int getPort() {
+ return config.getInteger(ConfigKeys.PORT);
+ }
+
+ public String getFlinkHost() {
+ return config.getString(ConfigKeys.FLINK_HOST);
+ }
+
+ public int getFlinkPort() {
+ return config.getInteger(ConfigKeys.FLINK_PORT);
+ }
+
+ public String getElasticsearchHost() {
+ return config.getString(ConfigKeys.ELASTIC_HOST);
+ }
+
+ public int getElasticsearchPort() {
+ return config.getInteger(ConfigKeys.ELASTIC_PORT);
+ }
+
+
+ public static final String iconBaseUrl = "http://" + StatisticsFlinkConfig.INSTANCE.getIconHost() + ":" + StatisticsFlinkConfig.INSTANCE.getIconPort() + "/assets/img/pe_icons";
+
+ public static final String getIconUrl(String pictureName) {
+ return iconBaseUrl + "/" + pictureName + ".png";
+ }
+
+ public String getIconHost() {
+ return config.getString(ConfigKeys.ICON_HOST);
+ }
+
+ public int getIconPort() {
+ return config.getInteger(ConfigKeys.ICON_PORT);
+ }
+
+ public int getElasticsearchPortRest() {
+ return config.getInteger(ConfigKeys.ELASTIC_PORT_REST);
+ }
+
+
+ public boolean getDebug() {
+ return config.getBoolean(ConfigKeys.DEBUG);
+ }
+
+ @Override
+ public String getId() {
+ return service_id;
+ }
+
+ @Override
+ public String getName() {
+ return config.getString(ConfigKeys.SERVICE_NAME);
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/MapKeySelector.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/MapKeySelector.java
new file mode 100644
index 0000000..ce655e3
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/MapKeySelector.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.extensions;
+
+import org.apache.flink.api.java.functions.KeySelector;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class MapKeySelector implements Serializable {
+
+ private String groupBy;
+
+ public MapKeySelector(String groupBy) {
+ this.groupBy = groupBy;
+ }
+
+ public KeySelector<Map<String, Object>, String> getKeySelector() {
+ return new KeySelector<Map<String, Object>, String>() {
+ @Override
+ public String getKey(Map<String, Object> in) throws Exception {
+ return String.valueOf(in.get(groupBy));
+ }
+ };
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/SlidingBatchWindow.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/SlidingBatchWindow.java
new file mode 100644
index 0000000..d0d2787
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/SlidingBatchWindow.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.extensions;
+
+import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class SlidingBatchWindow<IN> extends AbstractStreamOperator<List<IN>> implements
+ OneInputStreamOperator<IN, List<IN>> {
+
+ private Integer windowSize;
+ private List<IN> currentEvents;
+
+ public SlidingBatchWindow(Integer windowSize) {
+ super();
+ this.windowSize = windowSize;
+ this.currentEvents = new ArrayList<>();
+ }
+
+ @Override
+ public void processElement(StreamRecord<IN> in) throws Exception {
+ currentEvents.add(in.getValue());
+ if (currentEvents.size() > windowSize) {
+ currentEvents.remove(0);
+ }
+
+ output.collect(new StreamRecord<>(currentEvents, System.currentTimeMillis()));
+
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/SlidingEventTimeWindow.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/SlidingEventTimeWindow.java
new file mode 100644
index 0000000..2979dc4
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/SlidingEventTimeWindow.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.extensions;
+
+import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
+import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
+import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+public class SlidingEventTimeWindow<IN> extends AbstractUdfStreamOperator<List<IN>,
+ TimestampMappingFunction<IN>>
+ implements
+ OneInputStreamOperator<IN, List<IN>>, Serializable {
+
+ private Long timeWindowSizeInMillis;
+ private List<IN> currentEvents;
+
+ public SlidingEventTimeWindow(Long time, TimeUnit timeUnit, TimestampMappingFunction<IN>
+ timestampMappingFunction) {
+ super(timestampMappingFunction);
+ this.timeWindowSizeInMillis = toMilliseconds(time, timeUnit);
+ this.currentEvents = new ArrayList<>();
+
+ }
+
+ private Long toMilliseconds(Long time, TimeUnit timeUnit) {
+ return timeUnit.toMillis(time);
+ }
+
+ @Override
+ public void processElement(StreamRecord<IN> in) throws Exception {
+ Long currentTimestamp = userFunction.getTimestamp(in.getValue());
+
+ checkForRemoval(currentTimestamp);
+ currentEvents.add(in.getValue());
+
+ output.collect(new StreamRecord<>(currentEvents, System.currentTimeMillis()));
+
+ }
+
+ private void checkForRemoval(Long currentTimestamp) {
+ Iterator<IN> it = currentEvents.iterator();
+
+ while(it.hasNext()) {
+ IN next = it.next();
+ if (removalRequired(userFunction.getTimestamp(next), currentTimestamp)) {
+ it.remove();
+ } else {
+ break;
+ }
+ }
+ }
+
+ private boolean removalRequired(Long oldTimestamp, Long currentTimestamp) {
+ return (currentTimestamp - oldTimestamp > timeWindowSizeInMillis);
+ }
+
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/TimestampMappingFunction.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/TimestampMappingFunction.java
new file mode 100644
index 0000000..0d22d34
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/extensions/TimestampMappingFunction.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.extensions;
+
+import org.apache.flink.api.common.functions.Function;
+
+import java.io.Serializable;
+
+public interface TimestampMappingFunction<IN> extends Function, Serializable {
+
+ Long getTimestamp(IN in);
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryCalculator.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryCalculator.java
new file mode 100644
index 0000000..1f44aa6
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryCalculator.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class StatisticsSummaryCalculator implements FlatMapFunction<Map<String, Object>, Map<String, Object>> {
+
+ private String listPropertyName;
+
+ public StatisticsSummaryCalculator(String listPropertyName) {
+ this.listPropertyName = listPropertyName;
+ }
+
+ @Override
+ public void flatMap(Map<String, Object> in, Collector<Map<String, Object>> out) throws
+ Exception {
+ List<Double> listValues = ((List<Object>) in
+ .get(listPropertyName))
+ .stream()
+ .map(o -> Double.parseDouble(o.toString()))
+ .collect(Collectors.toList());
+
+ SummaryStatistics stats = new SummaryStatistics();
+
+ listValues.forEach(lv -> stats.addValue(lv));
+
+ in.put(StatisticsSummaryController.MIN, stats.getMin());
+ in.put(StatisticsSummaryController.MAX, stats.getMax());
+ in.put(StatisticsSummaryController.MEAN, stats.getMean());
+ in.put(StatisticsSummaryController.N, stats.getN());
+ in.put(StatisticsSummaryController.SUM, stats.getSum());
+ in.put(StatisticsSummaryController.STDDEV, stats.getStandardDeviation());
+ in.put(StatisticsSummaryController.VARIANCE, stats.getVariance());
+
+ out.collect(in);
+
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryController.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryController.java
new file mode 100644
index 0000000..d85cb5f
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryController.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat;
+
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.statistics.flink.config.StatisticsFlinkConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.sdk.utils.Datatypes;
+import org.streampipes.vocabulary.Statistics;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class StatisticsSummaryController extends FlinkDataProcessorDeclarer<StatisticsSummaryParameters> {
+
+ private static final String listPropertyMappingName = "list-property";
+
+ public static final String MIN = "min";
+ public static final String MAX = "max";
+ public static final String SUM = "sum";
+ public static final String STDDEV = "stddev";
+ public static final String VARIANCE = "variance";
+ public static final String MEAN = "mean";
+ public static final String N = "n";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.statistics.flink.statistics-summary", "Statistics Summary", "Calculate" +
+ " simple descriptive summary statistics")
+ .iconUrl(StatisticsFlinkConfig.getIconUrl("statistical_summary"))
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.listRequirement(Datatypes
+ .Number), Labels.from(listPropertyMappingName, "Property", "Select a list property"), PropertyScope.MEASUREMENT_PROPERTY)
+ .build())
+ .outputStrategy(OutputStrategies.append(EpProperties.doubleEp(Labels.empty(), MEAN, Statistics
+ .MEAN),
+ EpProperties.doubleEp(Labels.empty(), MIN, Statistics.MIN),
+ EpProperties.doubleEp(Labels.empty(), MAX, Statistics.MAX),
+ EpProperties.doubleEp(Labels.empty(), SUM, Statistics.SUM),
+ EpProperties.doubleEp(Labels.empty(), STDDEV, Statistics.STDDEV),
+ EpProperties.doubleEp(Labels.empty(), VARIANCE, Statistics.VARIANCE),
+ EpProperties.doubleEp(Labels.empty(), N, Statistics.N)))
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<StatisticsSummaryParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ String listPropertyMapping = extractor.mappingPropertyValue(listPropertyMappingName);
+
+ StatisticsSummaryParameters params = new StatisticsSummaryParameters(graph, listPropertyMapping);
+
+ return new StatisticsSummaryProgram(params);
+
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryParameters.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryParameters.java
new file mode 100644
index 0000000..b850cdc
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryParameters.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class StatisticsSummaryParameters extends EventProcessorBindingParams {
+
+ private String listPropertyName;
+
+ public StatisticsSummaryParameters(DataProcessorInvocation graph, String listPropertyName) {
+ super(graph);
+ this.listPropertyName = listPropertyName;
+ }
+
+ public String getListPropertyName() {
+ return listPropertyName;
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryProgram.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryProgram.java
new file mode 100644
index 0000000..df8fdbc
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/StatisticsSummaryProgram.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processors.statistics.flink.AbstractStatisticsProgram;
+
+import java.util.Map;
+
+public class StatisticsSummaryProgram extends AbstractStatisticsProgram<StatisticsSummaryParameters> {
+
+ public StatisticsSummaryProgram(StatisticsSummaryParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ public StatisticsSummaryProgram(StatisticsSummaryParameters params) {
+ super(params);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... messageStream) {
+ return messageStream[0].flatMap(new StatisticsSummaryCalculator(bindingParams.getListPropertyName()));
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryCalculatorWindow.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryCalculatorWindow.java
new file mode 100644
index 0000000..20fd4c7
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryCalculatorWindow.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat.window;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+import org.streampipes.processors.statistics.flink.processor.stat.StatisticsSummaryController;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class StatisticsSummaryCalculatorWindow implements FlatMapFunction<List<Map<String,
+ Object>>, Map<String, Object>>, Serializable {
+
+ private String partitionMapping;
+ private String valueToObserveMapping;
+
+ public StatisticsSummaryCalculatorWindow(String partitionMapping, String valueToObserveMapping) {
+ this.partitionMapping = partitionMapping;
+ this.valueToObserveMapping = valueToObserveMapping;
+ }
+
+ @Override
+ public void flatMap(List<Map<String, Object>> in, Collector<Map<String, Object>> out)
+ throws Exception {
+ List<Double> listValues = (in.stream().map(m -> Double.parseDouble(String.valueOf(m.get
+ (valueToObserveMapping))))
+ .collect(Collectors.toList()));
+
+ SummaryStatistics stats = new SummaryStatistics();
+
+ listValues.forEach(lv -> stats.addValue(lv));
+
+ Map<String, Object> outMap = new HashMap<>();
+
+ outMap.put("timestamp", System.currentTimeMillis());
+ outMap.put("id", in.get(in.size() - 1).get(partitionMapping));
+ outMap.put(StatisticsSummaryController.MIN, stats.getMin());
+ outMap.put(StatisticsSummaryController.MAX, stats.getMax());
+ outMap.put(StatisticsSummaryController.MEAN, stats.getMean());
+ outMap.put(StatisticsSummaryController.N, stats.getN());
+ outMap.put(StatisticsSummaryController.SUM, stats.getSum());
+ outMap.put(StatisticsSummaryController.STDDEV, stats.getStandardDeviation());
+ outMap.put(StatisticsSummaryController.VARIANCE, stats.getVariance());
+
+ out.collect(outMap);
+ }
+
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryControllerWindow.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryControllerWindow.java
new file mode 100644
index 0000000..926058c
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryControllerWindow.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat.window;
+
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.statistics.flink.config.StatisticsFlinkConfig;
+import org.streampipes.processors.statistics.flink.processor.stat.StatisticsSummaryController;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.vocabulary.Statistics;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+import java.util.concurrent.TimeUnit;
+
+public class StatisticsSummaryControllerWindow extends
+ FlinkDataProcessorDeclarer<StatisticsSummaryParametersWindow> {
+
+ private static final String VALUE_TO_OBSERVE = "value-to-observe";
+ private static final String PARTITION_BY = "partition-by";
+ private static final String TIMESTAMP_MAPPING = "timestamp-mapping";
+ private static final String TIME_WINDOW = "time-window";
+ private static final String TIME_SCALE = "time-scale";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.statistics.flink.statistics-summary-window", "Sliding Descriptive " +
+ "Statistics",
+ "Calculate" +
+ " simple descriptive summary statistics based on a configurable time window")
+ .iconUrl(StatisticsFlinkConfig.getIconUrl("statistics-icon"))
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.numberReq(),
+ Labels.from(VALUE_TO_OBSERVE, "Value to " +
+ "observe", "Provide a value where statistics are calculated upon"), PropertyScope.MEASUREMENT_PROPERTY)
+ .requiredPropertyWithUnaryMapping(EpRequirements.timestampReq(),
+ Labels.from(TIMESTAMP_MAPPING, "Time", "Provide a time parameter"), PropertyScope.HEADER_PROPERTY)
+ .requiredPropertyWithUnaryMapping(EpRequirements.stringReq(),
+ Labels.from(PARTITION_BY, "Group by", "Partition the stream by a given id"), PropertyScope.DIMENSION_PROPERTY)
+ .build())
+ .requiredIntegerParameter(Labels.from(TIME_WINDOW, "Time Window Size", "Size of the time window"))
+ .requiredSingleValueSelection(Labels.from(TIME_SCALE, "Time Window Scale", ""),
+ Options.from("Hours", "Minutes", "Seconds"))
+ .outputStrategy(OutputStrategies.fixed(
+ EpProperties.timestampProperty("timestamp"),
+ EpProperties.stringEp(Labels.empty(), "id", "http://schema.org/id"),
+ EpProperties.doubleEp(Labels.empty(), StatisticsSummaryController.MEAN, Statistics.MEAN),
+ EpProperties.doubleEp(Labels.empty(), StatisticsSummaryController.MIN, Statistics.MIN),
+ EpProperties.doubleEp(Labels.empty(), StatisticsSummaryController.MAX, Statistics.MAX),
+ EpProperties.doubleEp(Labels.empty(), StatisticsSummaryController.SUM, Statistics.SUM),
+ EpProperties.doubleEp(Labels.empty(), StatisticsSummaryController.STDDEV, Statistics.STDDEV),
+ EpProperties.doubleEp(Labels.empty(), StatisticsSummaryController.VARIANCE, Statistics.VARIANCE),
+ EpProperties.doubleEp(Labels.empty(), StatisticsSummaryController.N, Statistics.N)))
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<StatisticsSummaryParametersWindow> getRuntime(DataProcessorInvocation sepa, ProcessingElementParameterExtractor extractor) {
+
+ String valueToObserve = extractor.mappingPropertyValue(VALUE_TO_OBSERVE);
+ String timestampMapping = extractor.mappingPropertyValue(TIMESTAMP_MAPPING);
+
+ String groupBy = extractor.mappingPropertyValue(PARTITION_BY);
+
+ int timeWindowSize = extractor.singleValueParameter(TIME_WINDOW, Integer.class);
+ String scale = extractor.selectedSingleValue(TIME_SCALE, String.class);
+
+ TimeUnit timeUnit;
+
+ if (scale.equals("Hours")) {
+ timeUnit = TimeUnit.HOURS;
+ }
+ else if (scale.equals("Minutes")) {
+ timeUnit = TimeUnit.MINUTES;
+ }
+ else {
+ timeUnit = TimeUnit.SECONDS;
+ }
+
+ StatisticsSummaryParametersWindow params = new StatisticsSummaryParametersWindow(sepa,
+ valueToObserve, timestampMapping, groupBy, (long) timeWindowSize, timeUnit);
+
+ StatisticsSummaryParamsSerializable serializableParams = new StatisticsSummaryParamsSerializable
+ (valueToObserve, timestampMapping, groupBy, (long) timeWindowSize, timeUnit);
+
+ return new StatisticsSummaryProgramWindow(params, serializableParams);
+
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryParametersWindow.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryParametersWindow.java
new file mode 100644
index 0000000..190a343
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryParametersWindow.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat.window;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+import java.util.concurrent.TimeUnit;
+
+public class StatisticsSummaryParametersWindow extends EventProcessorBindingParams {
+
+ private String valueToObserve;
+ private String timestampMapping;
+ private String groupBy;
+ private Long timeWindowSize;
+ private TimeUnit timeUnit;
+
+
+
+ public StatisticsSummaryParametersWindow(DataProcessorInvocation graph) {
+ super(graph);
+ }
+
+ public StatisticsSummaryParametersWindow(DataProcessorInvocation graph, String valueToObserve,
+ String timestampMapping, String groupBy, Long
+ timeWindowSize, TimeUnit timeUnit) {
+ super(graph);
+ this.valueToObserve = valueToObserve;
+ this.timestampMapping = timestampMapping;
+ this.groupBy = groupBy;
+ this.timeWindowSize = timeWindowSize;
+ this.timeUnit = timeUnit;
+ }
+
+ public String getValueToObserve() {
+ return valueToObserve;
+ }
+
+ public String getGroupBy() {
+ return groupBy;
+ }
+
+ public Long getTimeWindowSize() {
+ return timeWindowSize;
+ }
+
+ public TimeUnit getTimeUnit() {
+ return timeUnit;
+ }
+
+ public String getTimestampMapping() {
+ return timestampMapping;
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryParamsSerializable.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryParamsSerializable.java
new file mode 100644
index 0000000..9be2abf
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryParamsSerializable.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat.window;
+
+import java.io.Serializable;
+import java.util.concurrent.TimeUnit;
+
+public class StatisticsSummaryParamsSerializable implements Serializable {
+
+ private String valueToObserve;
+ private String timestampMapping;
+ private String groupBy;
+ private long timeWindowSize;
+ private TimeUnit timeUnit;
+
+ public StatisticsSummaryParamsSerializable(String valueToObserve, String timestampMapping,
+ String groupBy, Long timeWindowSize, TimeUnit
+ timeUnit) {
+ this.valueToObserve = valueToObserve;
+ this.timestampMapping = timestampMapping;
+ this.groupBy = groupBy;
+ this.timeWindowSize = timeWindowSize;
+ this.timeUnit = timeUnit;
+ }
+
+
+ public String getValueToObserve() {
+ return valueToObserve;
+ }
+
+ public String getGroupBy() {
+ return groupBy;
+ }
+
+ public Long getTimeWindowSize() {
+ return timeWindowSize;
+ }
+
+ public TimeUnit getTimeUnit() {
+ return timeUnit;
+ }
+
+ public String getTimestampMapping() {
+ return timestampMapping;
+ }
+}
diff --git a/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryProgramWindow.java b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryProgramWindow.java
new file mode 100644
index 0000000..6559570
--- /dev/null
+++ b/streampipes-processors-statistics-flink/src/main/java/org/streampipes/processors/statistics/flink/processor/stat/window/StatisticsSummaryProgramWindow.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.statistics.flink.processor.stat.window;
+
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.streaming.api.TimeCharacteristic;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processors.statistics.flink.AbstractStatisticsProgram;
+import org.streampipes.processors.statistics.flink.extensions.MapKeySelector;
+import org.streampipes.processors.statistics.flink.extensions.SlidingEventTimeWindow;
+import org.streampipes.processors.statistics.flink.extensions.TimestampMappingFunction;
+
+import java.util.List;
+import java.util.Map;
+
+public class StatisticsSummaryProgramWindow extends
+ AbstractStatisticsProgram<StatisticsSummaryParametersWindow> {
+
+ private StatisticsSummaryParamsSerializable serializableParams;
+
+ public StatisticsSummaryProgramWindow(StatisticsSummaryParametersWindow params, StatisticsSummaryParamsSerializable serializableParams, boolean debug) {
+ super(params, debug);
+ this.streamTimeCharacteristic = TimeCharacteristic.EventTime;
+ this.serializableParams = serializableParams;
+ }
+
+ public StatisticsSummaryProgramWindow(StatisticsSummaryParametersWindow params, StatisticsSummaryParamsSerializable serializableParams) {
+ super(params);
+ this.streamTimeCharacteristic = TimeCharacteristic.EventTime;
+ this.serializableParams = serializableParams;
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... messageStream) {
+
+ StatisticsSummaryParamsSerializable sp = new
+ StatisticsSummaryParamsSerializable(serializableParams.getValueToObserve(),
+ serializableParams.getTimestampMapping(), serializableParams.getGroupBy(),
+ serializableParams.getTimeWindowSize(), serializableParams.getTimeUnit());
+ DataStream<Map<String, Object>> output = messageStream[0]
+ .keyBy(new MapKeySelector(sp.getGroupBy()).getKeySelector())
+ .transform
+ ("sliding-window-event-shift",
+ TypeInformation.of(new TypeHint<List<Map<String, Object>>>() {
+ }), new SlidingEventTimeWindow<>(sp.getTimeWindowSize(), sp.getTimeUnit(),
+ (TimestampMappingFunction<Map<String, Object>>) in ->
+ Long.parseLong(String.valueOf(in.get(sp.getTimestampMapping())))))
+ .flatMap(new StatisticsSummaryCalculatorWindow(sp.getGroupBy(), sp.getValueToObserve()));
+
+ return output;
+ }
+
+
+
+
+}
\ No newline at end of file
diff --git a/streampipes-processors-text-mining-flink/development/.env b/streampipes-processors-text-mining-flink/development/.env
new file mode 100644
index 0000000..40a4aef
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/development/.env
@@ -0,0 +1,5 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6050
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_FLINK_DEBUG=true
diff --git a/streampipes-processors-text-mining-flink/pom.xml b/streampipes-processors-text-mining-flink/pom.xml
new file mode 100644
index 0000000..cf112c8
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/pom.xml
@@ -0,0 +1,166 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-processors-text-mining-flink</artifactId>
+
+ <properties>
+ <opennlp.version>1.9.0</opennlp.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ <version>${streampipes.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-commons</artifactId>
+ <version>${streampipes.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-flink</artifactId>
+ <version>${streampipes.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.9.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.8.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.24</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ <version>${streampipes.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-cep-scala_2.11</artifactId>
+ <version>1.4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.optimaize.languagedetector</groupId>
+ <artifactId>language-detector</artifactId>
+ <version>0.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.opennlp</groupId>
+ <artifactId>opennlp-tools</artifactId>
+ <version>${opennlp.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.flinkspector</groupId>
+ <artifactId>flinkspector-datastream_2.11</artifactId>
+ <version>0.8.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.streampipes.processors.aggregation.flink.AggregationFlinkInit</mainClass>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <filters>
+
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml
+ </exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>streampipes-processors-textmining-flink</finalName>
+
+ </build>
+
+
+</project>
\ No newline at end of file
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/AbstractTextMiningProgram.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/AbstractTextMiningProgram.java
new file mode 100644
index 0000000..50f083e
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/AbstractTextMiningProgram.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.textmining.flink;
+
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import org.streampipes.processors.textmining.flink.config.TextMiningFlinkConfig;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public abstract class AbstractTextMiningProgram<B extends EventProcessorBindingParams> extends FlinkDataProcessorRuntime<B> {
+
+ public AbstractTextMiningProgram(B params, boolean debug) {
+ super(params, debug);
+ }
+
+ public AbstractTextMiningProgram(B params) {
+ super(params, false);
+ }
+
+ @Override
+ protected FlinkDeploymentConfig getDeploymentConfig() {
+ return new FlinkDeploymentConfig(TextMiningFlinkConfig.JAR_FILE,
+ TextMiningFlinkConfig.INSTANCE.getFlinkHost(), TextMiningFlinkConfig.INSTANCE.getFlinkPort());
+ }
+
+}
+
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/TextMiningFlinkInit.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/TextMiningFlinkInit.java
new file mode 100644
index 0000000..6f7d8e6
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/TextMiningFlinkInit.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2017 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.streampipes.processors.textmining.flink;
+
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.processors.textmining.flink.config.TextMiningFlinkConfig;
+import org.streampipes.processors.textmining.flink.processor.wordcount.WordCountController;
+
+public class TextMiningFlinkInit extends StandaloneModelSubmitter {
+
+ public static void main(String[] args) {
+ DeclarersSingleton.getInstance()
+ .add(new WordCountController());
+
+ new TextMiningFlinkInit().init(TextMiningFlinkConfig.INSTANCE);
+ }
+
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/config/ConfigKeys.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/config/ConfigKeys.java
new file mode 100644
index 0000000..a5977a4
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/config/ConfigKeys.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.textmining.flink.config;
+
+public class ConfigKeys {
+ final static String HOST = "SP_HOST";
+ final static String PORT = "SP_PORT";
+ final static String FLINK_HOST = "SP_FLINK_HOST";
+ final static String FLINK_PORT = "SP_FLINK_PORT";
+ final static String ICON_HOST = "SP_ICON_HOST";
+ final static String ICON_PORT = "SP_ICON_PORT";
+ final static String SERVICE_NAME = "SP_SERVICE_NAME";
+ final static String DEBUG = "SP_FLINK_DEBUG";
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/config/TextMiningFlinkConfig.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/config/TextMiningFlinkConfig.java
new file mode 100644
index 0000000..c96afaa
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/config/TextMiningFlinkConfig.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2017 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.streampipes.processors.textmining.flink.config;
+
+import org.streampipes.config.SpConfig;
+import org.streampipes.container.model.PeConfig;
+
+public enum TextMiningFlinkConfig implements PeConfig {
+ INSTANCE;
+
+ private SpConfig config;
+ public static final String JAR_FILE = "./streampipes-processors-textmining-flink.jar";
+
+
+ private final static String service_id = "pe/org.streampipes.processors.textmining.flink";
+ private final static String service_name = "Processors Text Mining Flink";
+ private final static String service_container_name = "processors-textmining-flink";
+
+
+ TextMiningFlinkConfig() {
+ config = SpConfig.getSpConfig(service_id);
+
+ config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
+ config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
+ config.register(ConfigKeys.FLINK_HOST, "jobmanager", "Host for the flink cluster");
+ config.register(ConfigKeys.FLINK_PORT, 6123, "Port for the flink cluster");
+
+ config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host");
+ config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx");
+
+ config.register(ConfigKeys.DEBUG, false, "When set to true programs are not deployed to cluster, but executed locally");
+
+ config.register(ConfigKeys.SERVICE_NAME, service_name, "The name of the service");
+
+ }
+
+ @Override
+ public String getHost() {
+ return config.getString(ConfigKeys.HOST);
+ }
+
+ @Override
+ public int getPort() {
+ return config.getInteger(ConfigKeys.PORT);
+ }
+
+ public String getFlinkHost() {
+ return config.getString(ConfigKeys.FLINK_HOST);
+ }
+
+ public int getFlinkPort() {
+ return config.getInteger(ConfigKeys.FLINK_PORT);
+ }
+
+ public static final String iconBaseUrl = "http://" + TextMiningFlinkConfig.INSTANCE.getIconHost() + ":" + TextMiningFlinkConfig.INSTANCE.getIconPort() + "/assets/img/pe_icons";
+
+ public static final String getIconUrl(String pictureName) {
+ return iconBaseUrl + "/" + pictureName + ".png";
+ }
+
+ public String getIconHost() {
+ return config.getString(ConfigKeys.ICON_HOST);
+ }
+
+ public int getIconPort() {
+ return config.getInteger(ConfigKeys.ICON_PORT);
+ }
+
+ public boolean getDebug() {
+ return config.getBoolean(ConfigKeys.DEBUG);
+ }
+
+ @Override
+ public String getId() {
+ return service_id;
+ }
+
+ @Override
+ public String getName() {
+ return config.getString(ConfigKeys.SERVICE_NAME);
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/entity/EntityExtraction.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/entity/EntityExtraction.java
new file mode 100644
index 0000000..cdb8a5a
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/entity/EntityExtraction.java
@@ -0,0 +1,58 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.entity;
+
+import opennlp.tools.namefind.NameFinderME;
+import opennlp.tools.namefind.TokenNameFinderModel;
+import opennlp.tools.util.Span;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+
+public class EntityExtraction implements FlatMapFunction<Map<String, Object>, Map<String, Object>> {
+
+ private String fieldName;
+ private EntityExtractionModel entityExtractionModel;
+ NameFinderME nameFinder;
+
+ public EntityExtraction(String fieldName, EntityExtractionModel entityExtractionModel) throws IOException {
+ this.fieldName = fieldName;
+ this.entityExtractionModel = entityExtractionModel;
+ this.loadModel();
+ }
+
+ private void loadModel() throws IOException {
+ try (InputStream modelIn = new FileInputStream(entityExtractionModel.getFilename())){
+ TokenNameFinderModel model = new TokenNameFinderModel(modelIn);
+ this.nameFinder = new NameFinderME(model);
+ }
+ }
+
+ @Override
+ public void flatMap(Map<String, Object> in, Collector<Map<String, Object>> out) throws Exception {
+ String value = String.valueOf(in.get(fieldName));
+ Span[] entities = nameFinder.find(new String[] {value});
+
+ // TODO
+
+ out.collect(in);
+
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/entity/EntityExtractionModel.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/entity/EntityExtractionModel.java
new file mode 100644
index 0000000..82a9442
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/entity/EntityExtractionModel.java
@@ -0,0 +1,40 @@
+package org.streampipes.processors.textmining.flink.processor.entity;
+
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+public enum EntityExtractionModel {
+
+ PERSON("Person", "en-ner-location.bin"),
+ ORGANIZATION("Organization", "en-ner-organization.bin"),
+ LOCATION("Location", "en-ner-location.bin");
+
+ private String modelName;
+ private String filename;
+
+ EntityExtractionModel(String modelName, String filename) {
+ this.modelName = modelName;
+ this.filename = filename;
+ }
+
+ public String getModelName() {
+ return modelName;
+ }
+
+ public String getFilename() {
+ return filename;
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetection.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetection.java
new file mode 100644
index 0000000..9de488b
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetection.java
@@ -0,0 +1,72 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.language;
+
+import com.optimaize.langdetect.LanguageDetector;
+import com.optimaize.langdetect.LanguageDetectorBuilder;
+import com.optimaize.langdetect.i18n.LdLocale;
+import com.optimaize.langdetect.ngram.NgramExtractors;
+import com.optimaize.langdetect.profiles.LanguageProfile;
+import com.optimaize.langdetect.profiles.LanguageProfileReader;
+import com.optimaize.langdetect.text.CommonTextObjectFactories;
+import com.optimaize.langdetect.text.TextObject;
+import com.optimaize.langdetect.text.TextObjectFactory;
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class LanguageDetection implements FlatMapFunction<Map<String, Object>, Map<String, Object>> {
+
+ private static final String LANGUAGE_KEY = "language";
+
+ private String fieldName;
+ private LanguageDetector languageDetector;
+ private TextObjectFactory textObjectFactory;
+
+ public LanguageDetection(String fieldName) {
+ this.fieldName = fieldName;
+ List<LanguageProfile> languageProfiles = null;
+ try {
+ languageProfiles = new LanguageProfileReader().readAllBuiltIn();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ this.languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
+ .withProfiles(languageProfiles)
+ .build();
+
+ this.textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
+ }
+
+ @Override
+ public void flatMap(Map<String, Object> in, Collector<Map<String, Object>> out) {
+
+ TextObject textObject = textObjectFactory.forText(String.valueOf(in.get(fieldName)));
+ com.google.common.base.Optional<LdLocale> lang = languageDetector.detect(textObject);
+
+ if (lang.isPresent()) {
+ in.put(LANGUAGE_KEY, lang.get().getLanguage());
+ } else {
+ in.put(LANGUAGE_KEY, "unknown");
+ }
+
+ out.collect(in);
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionController.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionController.java
new file mode 100644
index 0000000..8e7f383
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionController.java
@@ -0,0 +1,66 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.language;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.textmining.flink.config.TextMiningFlinkConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class LanguageDetectionController extends FlinkDataProcessorDeclarer<LanguageDetectionParameters> {
+
+ private static final String RESOURCE_ID = "strings.languagedetection";
+ private static final String PE_ID = "org.streampipes.processors.textmining.flink.languagedetection";
+
+ private static final String DETECTION_FIELD_KEY = "detectionField";
+ private static final String LANGUAGE_KEY = "language";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create(getLabel(PE_ID))
+ .category(DataProcessorType.ENRICH_TEXT)
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements.stringReq(),
+ getLabel(DETECTION_FIELD_KEY),
+ PropertyScope.NONE)
+ .build())
+ .outputStrategy(OutputStrategies.append(EpProperties.stringEp(
+ getLabel(LANGUAGE_KEY),
+ "language",
+ "http://schema.org/language")))
+ .build();
+ }
+
+ private Label getLabel(String id) {
+ return Labels.fromResources(RESOURCE_ID, id);
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<LanguageDetectionParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ String fieldName = extractor.mappingPropertyValue(DETECTION_FIELD_KEY);
+
+ return new LanguageDetectionProgram(new LanguageDetectionParameters(graph, fieldName), TextMiningFlinkConfig.INSTANCE.getDebug());
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionParameters.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionParameters.java
new file mode 100644
index 0000000..0fe438e
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionParameters.java
@@ -0,0 +1,35 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.language;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class LanguageDetectionParameters extends EventProcessorBindingParams {
+
+ private String fieldName;
+
+ public LanguageDetectionParameters(DataProcessorInvocation graph, String fieldName)
+ {
+ super(graph);
+ this.fieldName = fieldName;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionProgram.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionProgram.java
new file mode 100644
index 0000000..e701a6e
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/language/LanguageDetectionProgram.java
@@ -0,0 +1,38 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.language;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processors.textmining.flink.AbstractTextMiningProgram;
+
+import java.util.Map;
+
+public class LanguageDetectionProgram extends AbstractTextMiningProgram<LanguageDetectionParameters> {
+
+ public LanguageDetectionProgram(LanguageDetectionParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ public LanguageDetectionProgram(LanguageDetectionParameters params) {
+ super(params);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... messageStream) {
+ return messageStream[0]
+ .flatMap(new LanguageDetection(params.getFieldName()));
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetection.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetection.java
new file mode 100644
index 0000000..66a0f70
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetection.java
@@ -0,0 +1,19 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.sentiment;
+
+public class SentimentDetection {
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionController.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionController.java
new file mode 100644
index 0000000..0210160
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionController.java
@@ -0,0 +1,19 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.sentiment;
+
+public class SentimentDetectionController {
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionParameters.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionParameters.java
new file mode 100644
index 0000000..c9d7e87
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionParameters.java
@@ -0,0 +1,19 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.sentiment;
+
+public class SentimentDetectionParameters {
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionProgram.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionProgram.java
new file mode 100644
index 0000000..350eedc
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/sentiment/SentimentDetectionProgram.java
@@ -0,0 +1,19 @@
+/*
+Copyright 2018 FZI Forschungszentrum Informatik
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package org.streampipes.processors.textmining.flink.processor.sentiment;
+
+public class SentimentDetectionProgram {
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/Word.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/Word.java
new file mode 100644
index 0000000..5264853
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/Word.java
@@ -0,0 +1,39 @@
+package org.streampipes.processors.textmining.flink.processor.wordcount;
+
+public class Word {
+
+ private String word;
+ private int count;
+
+ public Word() {
+
+ }
+
+ public Word(String word, int count) {
+ super();
+ this.word = word;
+ this.count = count;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+
+
+
+
+
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountController.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountController.java
new file mode 100644
index 0000000..b9f0ea4
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountController.java
@@ -0,0 +1,57 @@
+package org.streampipes.processors.textmining.flink.processor.wordcount;
+
+import org.streampipes.model.DataProcessorType;
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.textmining.flink.config.TextMiningFlinkConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class WordCountController extends FlinkDataProcessorDeclarer<WordCountParameters> {
+
+ private static final String RESOURCE_ID = "strings.wordcount";
+ private static final String PE_ID = "org.streampipes.processors.textmining.flink.wordcount";
+
+ private static final String WORD_COUNT_FIELD_KEY = "wordcountField";
+ private static final String TIME_WINDOW_KEY = "timeWindowKey";
+ private static final String WORD_KEY = "word";
+ private static final String COUNT_KEY = "count";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create(getLabel(PE_ID))
+ .category(DataProcessorType.AGGREGATE)
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(
+ EpRequirements.stringReq(),
+ getLabel(WORD_COUNT_FIELD_KEY),
+ PropertyScope.NONE)
+ .build())
+ .outputStrategy(OutputStrategies.fixed(EpProperties.stringEp(
+ getLabel(WORD_KEY),
+ "word",
+ "http://schema.org/text"), EpProperties.integerEp(getLabel(COUNT_KEY), "count", "http://schema.org/number")))
+ .requiredIntegerParameter(getLabel(TIME_WINDOW_KEY))
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<WordCountParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+
+ String fieldName = extractor.mappingPropertyValue(WORD_COUNT_FIELD_KEY);
+ Integer timeWindowValue = extractor.singleValueParameter(TIME_WINDOW_KEY, Integer.class);
+
+ return new WordCountProgram(new WordCountParameters(graph, fieldName, timeWindowValue), TextMiningFlinkConfig.INSTANCE.getDebug());
+
+ }
+
+ private Label getLabel(String id) {
+ return Labels.fromResources(RESOURCE_ID, id);
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountParameters.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountParameters.java
new file mode 100644
index 0000000..9ffd436
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountParameters.java
@@ -0,0 +1,25 @@
+package org.streampipes.processors.textmining.flink.processor.wordcount;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class WordCountParameters extends EventProcessorBindingParams {
+
+ private String wordCountFieldName;
+ private Integer timeWindowValue;
+
+ public WordCountParameters(DataProcessorInvocation graph, String wordCountFieldName, Integer timeWindowValue)
+ {
+ super(graph);
+ this.wordCountFieldName = wordCountFieldName;
+ this.timeWindowValue = timeWindowValue;
+ }
+
+ public String getWordCountFieldName() {
+ return wordCountFieldName;
+ }
+
+ public Integer getTimeWindowValue() {
+ return timeWindowValue;
+ }
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountProgram.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountProgram.java
new file mode 100644
index 0000000..854ae53
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordCountProgram.java
@@ -0,0 +1,31 @@
+package org.streampipes.processors.textmining.flink.processor.wordcount;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processors.textmining.flink.AbstractTextMiningProgram;
+import org.streampipes.wrapper.flink.converter.ObjectToMapConverter;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class WordCountProgram extends AbstractTextMiningProgram<WordCountParameters> implements Serializable {
+
+ public WordCountProgram(WordCountParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ public WordCountProgram(WordCountParameters params) {
+ super(params);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(
+ DataStream<Map<String, Object>>... messageStream) {
+
+ return messageStream[0]
+ .flatMap(new WordSplitter(params.getWordCountFieldName()))
+ .keyBy("word")
+ .sum("count")
+ .flatMap(new ObjectToMapConverter<>());
+ }
+
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordSplitter.java b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordSplitter.java
new file mode 100644
index 0000000..a09950c
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/java/org/streampipes/processors/textmining/flink/processor/wordcount/WordSplitter.java
@@ -0,0 +1,29 @@
+package org.streampipes.processors.textmining.flink.processor.wordcount;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+
+import java.util.Map;
+
+public class WordSplitter implements FlatMapFunction<Map<String, Object>, Word> {
+
+ private String mappingPropertyName;
+
+ public WordSplitter(String mappingPropertyName) {
+ this.mappingPropertyName = mappingPropertyName;
+ }
+
+ @Override
+ public void flatMap(Map<String, Object> in,
+ Collector<Word> out) throws Exception {
+
+ String propertyValue = (String) in.get(mappingPropertyName);
+ for(String word : propertyValue.split(" "))
+ {
+ out.collect(new Word(word, 1));
+ }
+ }
+
+
+
+}
diff --git a/streampipes-processors-text-mining-flink/src/main/resources/en-ner-location.bin b/streampipes-processors-text-mining-flink/src/main/resources/en-ner-location.bin
new file mode 100644
index 0000000..f3788bc
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/resources/en-ner-location.bin
Binary files differ
diff --git a/streampipes-processors-text-mining-flink/src/main/resources/en-ner-organization.bin b/streampipes-processors-text-mining-flink/src/main/resources/en-ner-organization.bin
new file mode 100644
index 0000000..1fb6d9f
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/resources/en-ner-organization.bin
Binary files differ
diff --git a/streampipes-processors-text-mining-flink/src/main/resources/en-ner-person.bin b/streampipes-processors-text-mining-flink/src/main/resources/en-ner-person.bin
new file mode 100644
index 0000000..2f68318
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/resources/en-ner-person.bin
Binary files differ
diff --git a/streampipes-processors-text-mining-flink/src/main/resources/strings.languagedetection b/streampipes-processors-text-mining-flink/src/main/resources/strings.languagedetection
new file mode 100644
index 0000000..faaf935
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/resources/strings.languagedetection
@@ -0,0 +1,7 @@
+languagedetection.title=Language Detection
+languagedetection.description=Detects the language of a written text.
+
+detectionField.title=Field Selection
+detectionField.description=Specifies the field where the language detection should be applied upon.
+
+
diff --git a/streampipes-processors-text-mining-flink/src/main/resources/strings.wordcount b/streampipes-processors-text-mining-flink/src/main/resources/strings.wordcount
new file mode 100644
index 0000000..bcb8b57
--- /dev/null
+++ b/streampipes-processors-text-mining-flink/src/main/resources/strings.wordcount
@@ -0,0 +1,10 @@
+org.streampipes.processors.textmining.flink.wordcount.title=Word Count
+org.streampipes.processors.textmining.flink.wordcount.description=Counts words on continuous text-based streams
+
+wordcountField.title=Property Selection
+wordcountField.description=Specifies the event property from your stream that should be counted.
+
+timeWindow.title=Time Window Size
+timeWindow.description=Size of the time window in seconds
+
+
diff --git a/streampipes-processors-transformation-flink/Dockerfile b/streampipes-processors-transformation-flink/Dockerfile
new file mode 100644
index 0000000..64e2472
--- /dev/null
+++ b/streampipes-processors-transformation-flink/Dockerfile
@@ -0,0 +1,8 @@
+FROM anapsix/alpine-java
+
+EXPOSE 8090
+ENV CONSUL_LOCATION consul
+
+ADD ./target/streampipes-processors-transformation-flink.jar /streampipes-processors-transformation-flink.jar
+
+ENTRYPOINT ["java", "-jar", "/streampipes-processors-transformation-flink.jar"]
diff --git a/streampipes-processors-transformation-flink/deployment/docker-compose.yml b/streampipes-processors-transformation-flink/deployment/docker-compose.yml
new file mode 100644
index 0000000..3dd0d53
--- /dev/null
+++ b/streampipes-processors-transformation-flink/deployment/docker-compose.yml
@@ -0,0 +1,13 @@
+version: "2"
+services:
+ processors-transformation-flink:
+ image: ${SP_DOCKER_REGISTRY}/streampipes/streampipes-pipeline-elements/processors-transformation-flink:${SP_PE_VERSION}
+ depends_on:
+ - "consul"
+# ports:
+# - "8098:8090"
+ environment:
+ - SP_ICON_HOST=${SP_ICON_HOST}
+ networks:
+ spnet:
+
diff --git a/streampipes-processors-transformation-flink/development/.env b/streampipes-processors-transformation-flink/development/.env
new file mode 100644
index 0000000..f8637f7
--- /dev/null
+++ b/streampipes-processors-transformation-flink/development/.env
@@ -0,0 +1,5 @@
+# Those parameters are used by IntelliJ to set the default consul parameters for development
+SP_PORT=6055
+SP_HOST=localhost
+SP_ICON_HOST=localhost
+SP_FLINK_DEBUG=true
diff --git a/streampipes-processors-transformation-flink/pom.xml b/streampipes-processors-transformation-flink/pom.xml
new file mode 100644
index 0000000..f2c45c6
--- /dev/null
+++ b/streampipes-processors-transformation-flink/pom.xml
@@ -0,0 +1,176 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>streampipes-pipeline-elements</artifactId>
+ <groupId>org.streampipes</groupId>
+ <version>0.60.0</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>streampipes-processors-transformation-flink</artifactId>
+
+ <properties>
+ <flink.version>1.4.0</flink.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-container-standalone</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-commons</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka-clients</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-wrapper-flink</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-sdk</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-measurement-units</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.9.3</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ <version>2.8.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.24</version>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-config</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>io.flinkspector</groupId>
+ <artifactId>flinkspector-datastream_2.11</artifactId>
+ <version>0.8.4</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.streampipes</groupId>
+ <artifactId>streampipes-test-utils</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <!-- Run shade goal on package phase -->
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <artifactSet>
+ <excludes>
+ <exclude>org.apache.flink:flink-java-examples</exclude>
+ <exclude>org.apache.flink:flink-scala-examples</exclude>
+ <exclude>org.apache.flink:flink-streaming-examples</exclude>
+
+ <!-- Also exclude very big transitive dependencies of Flink WARNING:
+ You have to remove these excludes if your code relies on other versions of
+ these dependencies. -->
+ <filters>
+
+
+ </filters>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>org.apache.flink:*</artifact>
+ <excludes>
+ <exclude>web-docs/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java/pom.xml</exclude>
+ <exclude>META-INF/maven/com.github.jsonld-java/jsonld-java-sesame/pom.xml</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <!-- add Main-Class to manifest file -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.streampipes.processors.transformation.flink.TransformationFlinkInit</mainClass>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+
+ </transformers>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <finalName>streampipes-processors-transformation-flink</finalName>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/AbstractFlinkTransformationProgram.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/AbstractFlinkTransformationProgram.java
new file mode 100644
index 0000000..0e381e0
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/AbstractFlinkTransformationProgram.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.transformation.flink;
+
+import org.streampipes.processors.transformation.flink.config.TransformationFlinkConfig;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+import org.streampipes.wrapper.flink.FlinkDeploymentConfig;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public abstract class AbstractFlinkTransformationProgram<B extends EventProcessorBindingParams> extends FlinkDataProcessorRuntime<B> {
+
+ public AbstractFlinkTransformationProgram(B params, boolean debug) {
+ super(params, debug);
+ }
+
+ public AbstractFlinkTransformationProgram(B params) {
+ super(params, TransformationFlinkConfig.INSTANCE.getDebug());
+ }
+
+ @Override
+ protected FlinkDeploymentConfig getDeploymentConfig() {
+ return new FlinkDeploymentConfig(TransformationFlinkConfig.JAR_FILE,
+ TransformationFlinkConfig.INSTANCE.getFlinkHost(), TransformationFlinkConfig.INSTANCE.getFlinkPort());
+ }
+
+
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/TransformationFlinkInit.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/TransformationFlinkInit.java
new file mode 100644
index 0000000..fd1ca8a
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/TransformationFlinkInit.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink;
+
+import org.streampipes.container.init.DeclarersSingleton;
+import org.streampipes.container.standalone.init.StandaloneModelSubmitter;
+import org.streampipes.processors.transformation.flink.config.TransformationFlinkConfig;
+import org.streampipes.processors.transformation.flink.processor.converter.FieldConverterController;
+import org.streampipes.processors.transformation.flink.processor.hasher.FieldHasherController;
+import org.streampipes.processors.transformation.flink.processor.mapper.FieldMapperController;
+import org.streampipes.processors.transformation.flink.processor.rename.FieldRenamerController;
+
+public class TransformationFlinkInit extends StandaloneModelSubmitter {
+
+ public static void main(String[] args) {
+ DeclarersSingleton.getInstance()
+ .add(new FieldConverterController())
+ .add(new FieldHasherController())
+ .add(new FieldMapperController())
+ //.add(new MeasurementUnitConverterController())
+ .add(new FieldRenamerController());
+
+ new TransformationFlinkInit().init(TransformationFlinkConfig.INSTANCE);
+ }
+
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/config/ConfigKeys.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/config/ConfigKeys.java
new file mode 100644
index 0000000..90ac12e
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/config/ConfigKeys.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.config;
+
+public class ConfigKeys {
+ final static String HOST = "SP_HOST";
+ final static String PORT = "SP_PORT";
+ final static String FLINK_HOST = "SP_FLINK_HOST";
+ final static String FLINK_PORT = "SP_FLINK_PORT";
+ final static String ICON_HOST = "SP_ICON_HOST";
+ final static String ICON_PORT = "SP_ICON_PORT";
+ final static String SERVICE_NAME = "SP_SERVICE_NAME";
+ final static String DEBUG = "SP_FLINK_DEBUG";
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
new file mode 100644
index 0000000..a9875b2
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/config/TransformationFlinkConfig.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2017 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.streampipes.processors.transformation.flink.config;
+
+import org.streampipes.config.SpConfig;
+import org.streampipes.container.model.PeConfig;
+
+public enum TransformationFlinkConfig implements PeConfig {
+ INSTANCE;
+
+ private SpConfig config;
+ public static final String JAR_FILE = "./streampipes-processors-transformation-flink.jar";
+
+ private final static String service_id = "pe/org.streampipes.processors.transformation.flink";
+ private final static String service_name = "Processors Transformation Flink";
+ private final static String service_container_name = "processors-transformation-flink";
+
+ TransformationFlinkConfig() {
+ config = SpConfig.getSpConfig(service_id);
+
+ config.register(ConfigKeys.HOST, service_container_name, "Hostname for the pe mixed flink component");
+ config.register(ConfigKeys.PORT, 8090, "Port for the pe mixed flink component");
+ config.register(ConfigKeys.FLINK_HOST, "jobmanager", "Host for the flink cluster");
+ config.register(ConfigKeys.FLINK_PORT, 6123, "Port for the flink cluster");
+
+ config.register(ConfigKeys.ICON_HOST, "backend", "Hostname for the icon host");
+ config.register(ConfigKeys.ICON_PORT, 80, "Port for the icons in nginx");
+
+ config.register(ConfigKeys.DEBUG, false, "When set to true programs are not deployed to cluster, but executed locally");
+
+ config.register(ConfigKeys.SERVICE_NAME, service_name, "The name of the service");
+
+ }
+
+ @Override
+ public String getHost() {
+ return config.getString(ConfigKeys.HOST);
+ }
+
+ @Override
+ public int getPort() {
+ return config.getInteger(ConfigKeys.PORT);
+ }
+
+ public String getFlinkHost() {
+ return config.getString(ConfigKeys.FLINK_HOST);
+ }
+
+ public int getFlinkPort() {
+ return config.getInteger(ConfigKeys.FLINK_PORT);
+ }
+
+
+ public static final String iconBaseUrl = "http://" + TransformationFlinkConfig.INSTANCE.getIconHost() + ":" + TransformationFlinkConfig.INSTANCE.getIconPort() + "/assets/img/pe_icons";
+
+ public static final String getIconUrl(String pictureName) {
+ return iconBaseUrl + "/" + pictureName + ".png";
+ }
+
+ public String getIconHost() {
+ return config.getString(ConfigKeys.ICON_HOST);
+ }
+
+ public int getIconPort() {
+ return config.getInteger(ConfigKeys.ICON_PORT);
+ }
+
+ public boolean getDebug() {
+ return config.getBoolean(ConfigKeys.DEBUG);
+ }
+
+ @Override
+ public String getId() {
+ return service_id;
+ }
+
+ @Override
+ public String getName() {
+ return config.getString(ConfigKeys.SERVICE_NAME);
+ }
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverter.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverter.java
new file mode 100644
index 0000000..44f113b
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverter.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.transformation.flink.processor.converter;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.streampipes.vocabulary.XSD;
+
+import java.util.Map;
+
+public class FieldConverter implements FlatMapFunction<Map<String, Object>, Map<String, Object>> {
+
+ private static Logger LOG = LoggerFactory.getLogger(FieldConverter.class);
+
+ private String convertProperty;
+ private String targetDatatype;
+
+ public FieldConverter(String convertProperty, String targetDatatype) {
+ this.convertProperty = convertProperty;
+ this.targetDatatype = targetDatatype;
+ }
+
+
+ @Override
+ public void flatMap(Map<String, Object> in, Collector<Map<String, Object>> out) {
+ String value = String.valueOf(in.get(convertProperty));
+ try {
+ if (targetDatatype.equals(XSD._float.toString())) {
+ in.put(convertProperty, Float.parseFloat(value));
+ } else {
+ in.put(convertProperty, Integer.parseInt(value));
+ }
+
+ out.collect(in);
+
+ } catch (NumberFormatException e) {
+ LOG.error("Field Converter could not convert value: " + value + " of event: " + in);
+ }
+
+ }
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterController.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterController.java
new file mode 100644
index 0000000..cf6d4df
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterController.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.transformation.flink.processor.converter;
+
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.transformation.flink.config.TransformationFlinkConfig;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.vocabulary.XSD;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class FieldConverterController extends
+ FlinkDataProcessorDeclarer<FieldConverterParameters> {
+
+ public static final String CONVERT_PROPERTY = "convert-property";
+ public static final String TARGET_TYPE = "target-type";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.transformation.flink.field-converter", "Field Converter",
+ "Converts a string value to a number data type")
+ .iconUrl(TransformationFlinkConfig.getIconUrl("field_converter"))
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.stringReq(), Labels.from
+ (CONVERT_PROPERTY,"Property", "The" +
+ " property to convert"), PropertyScope.NONE)
+ .build())
+ .requiredSingleValueSelection(Labels.from(TARGET_TYPE, "Datatype", "The target datatype"), Options.from
+ (new Tuple2<>("Float", XSD._float.toString()), new Tuple2<>
+ ("Integer", XSD._integer.toString())))
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .outputStrategy(OutputStrategies.transform(TransformOperations
+ .dynamicDatatypeTransformation(CONVERT_PROPERTY, TARGET_TYPE)))
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<FieldConverterParameters> getRuntime(DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ String convertProperty = extractor.mappingPropertyValue(CONVERT_PROPERTY);
+ String targetDatatype = extractor.selectedSingleValueInternalName(TARGET_TYPE, String.class);
+
+ FieldConverterParameters staticParams = new FieldConverterParameters(
+ graph,
+ convertProperty,
+ targetDatatype
+ );
+
+ return new FieldConverterProgram(staticParams);
+ }
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterParameters.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterParameters.java
new file mode 100644
index 0000000..c2c5e0e
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterParameters.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.transformation.flink.processor.converter;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class FieldConverterParameters extends EventProcessorBindingParams {
+
+ private String convertProperty;
+ private String targetDatatype;
+
+ public FieldConverterParameters(DataProcessorInvocation graph, String convertProperty, String targetDatatype) {
+ super(graph);
+
+ this.convertProperty = convertProperty;
+ this.targetDatatype = targetDatatype;
+ }
+
+ public String getConvertProperty() {
+ return convertProperty;
+ }
+
+ public String getTargetDatatype() {
+ return targetDatatype;
+ }
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterProgram.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterProgram.java
new file mode 100644
index 0000000..79331b1
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/converter/FieldConverterProgram.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.streampipes.processors.transformation.flink.processor.converter;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processors.transformation.flink.AbstractFlinkTransformationProgram;
+
+import java.util.Map;
+
+public class FieldConverterProgram extends AbstractFlinkTransformationProgram<FieldConverterParameters> {
+
+ public FieldConverterProgram(FieldConverterParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ public FieldConverterProgram(FieldConverterParameters params) {
+ super(params);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(DataStream<Map<String, Object>>... messageStream) {
+ return messageStream[0].flatMap(new FieldConverter(params.getConvertProperty(), params.getTargetDatatype()));
+ }
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasher.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasher.java
new file mode 100644
index 0000000..b7b359c
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasher.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.processor.hasher;
+
+import org.apache.flink.api.common.functions.FlatMapFunction;
+import org.apache.flink.util.Collector;
+import org.streampipes.processors.transformation.flink.processor.hasher.algorithm.HashAlgorithm;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class FieldHasher implements Serializable, FlatMapFunction<Map<String, Object>, Map<String, Object>>{
+
+ private HashAlgorithm hashAlgorithm;
+ private String propertyName;
+
+ public FieldHasher(String propertyName, HashAlgorithm hashAlgorithm) {
+ this.propertyName = propertyName;
+ this.hashAlgorithm = hashAlgorithm;
+ }
+
+ @Override
+ public void flatMap(Map<String, Object> in,
+ Collector<Map<String, Object>> out) throws Exception {
+ in.put(propertyName, hashAlgorithm.toHashValue(in.get(propertyName)));
+ out.collect(in);
+ }
+
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherController.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherController.java
new file mode 100644
index 0000000..c1782ef
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherController.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.processor.hasher;
+
+import org.streampipes.model.graph.DataProcessorDescription;
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.model.schema.PropertyScope;
+import org.streampipes.processors.transformation.flink.config.TransformationFlinkConfig;
+import org.streampipes.processors.transformation.flink.processor.hasher.algorithm.HashAlgorithmType;
+import org.streampipes.sdk.builder.ProcessingElementBuilder;
+import org.streampipes.sdk.builder.StreamRequirementsBuilder;
+import org.streampipes.sdk.extractor.ProcessingElementParameterExtractor;
+import org.streampipes.sdk.helpers.*;
+import org.streampipes.wrapper.flink.FlinkDataProcessorDeclarer;
+import org.streampipes.wrapper.flink.FlinkDataProcessorRuntime;
+
+public class FieldHasherController extends FlinkDataProcessorDeclarer<FieldHasherParameters> {
+
+ private static final String HASH_PROPERTIES = "property-mapping";
+ private static final String HASH_ALGORITHM = "hash-algorithm";
+
+ @Override
+ public DataProcessorDescription declareModel() {
+ return ProcessingElementBuilder.create("org.streampipes.processors.transformation.flink.fieldhasher", "Field Hasher",
+ "The Field Hasher uses an algorithm to encode values in a field. The Field Hasher can use MD5, SHA1 or SHA2 to hash field values.")
+ .requiredStream(StreamRequirementsBuilder
+ .create()
+ .requiredPropertyWithUnaryMapping(EpRequirements.anyProperty(), Labels.from
+ (HASH_PROPERTIES, "Field", "The field the hash function should be applied on"), PropertyScope.NONE)
+ .build())
+ .iconUrl(TransformationFlinkConfig.getIconUrl("field-hasher-icon"))
+ .requiredSingleValueSelection(Labels.from("hash-algorithm", "Hash Algorithm", "The hash algorithm that should be used."),
+ Options.from("SHA1", "SHA2", "MD5"))
+ .outputStrategy(OutputStrategies.keep())
+ .supportedFormats(SupportedFormats.jsonFormat())
+ .supportedProtocols(SupportedProtocols.kafka(), SupportedProtocols.jms())
+ .build();
+ }
+
+ @Override
+ public FlinkDataProcessorRuntime<FieldHasherParameters> getRuntime(
+ DataProcessorInvocation graph, ProcessingElementParameterExtractor extractor) {
+ String propertyName = extractor.mappingPropertyValue(HASH_PROPERTIES);
+
+ HashAlgorithmType hashAlgorithmType = HashAlgorithmType.valueOf(extractor.selectedSingleValue(HASH_ALGORITHM, String.class));
+
+ return new FieldHasherProgram(
+ new FieldHasherParameters(graph, propertyName, hashAlgorithmType));
+ }
+
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherParameters.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherParameters.java
new file mode 100644
index 0000000..41442d4
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherParameters.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.processor.hasher;
+
+import org.streampipes.model.graph.DataProcessorInvocation;
+import org.streampipes.processors.transformation.flink.processor.hasher.algorithm.HashAlgorithmType;
+import org.streampipes.wrapper.params.binding.EventProcessorBindingParams;
+
+public class FieldHasherParameters extends EventProcessorBindingParams {
+
+ private String propertyName;
+ private HashAlgorithmType hashAlgorithmType;
+
+ public FieldHasherParameters(DataProcessorInvocation graph, String propertyName, HashAlgorithmType hashAlgorithmType) {
+ super(graph);
+ this.propertyName = propertyName;
+ this.hashAlgorithmType = hashAlgorithmType;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+
+ public HashAlgorithmType getHashAlgorithmType() {
+ return hashAlgorithmType;
+ }
+
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherProgram.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherProgram.java
new file mode 100644
index 0000000..416f4d2
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/FieldHasherProgram.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.processor.hasher;
+
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.streampipes.processors.transformation.flink.AbstractFlinkTransformationProgram;
+
+import java.io.Serializable;
+import java.util.Map;
+
+public class FieldHasherProgram extends AbstractFlinkTransformationProgram<FieldHasherParameters>
+ implements Serializable{
+
+ public FieldHasherProgram(FieldHasherParameters params, boolean debug) {
+ super(params, debug);
+ }
+
+ public FieldHasherProgram(FieldHasherParameters params) {
+ super(params);
+ }
+
+ @Override
+ protected DataStream<Map<String, Object>> getApplicationLogic(
+ DataStream<Map<String, Object>>... messageStream) {
+ return messageStream[0].flatMap(new FieldHasher(bindingParams.getPropertyName(),
+ bindingParams.getHashAlgorithmType().hashAlgorithm()));
+ }
+
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/HashAlgorithm.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/HashAlgorithm.java
new file mode 100644
index 0000000..673583c
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/HashAlgorithm.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.processor.hasher.algorithm;
+
+import java.io.Serializable;
+
+public interface HashAlgorithm extends Serializable {
+
+ String toHashValue(Object value);
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/HashAlgorithmType.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/HashAlgorithmType.java
new file mode 100644
index 0000000..40deec4
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/HashAlgorithmType.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.processor.hasher.algorithm;
+
+public enum HashAlgorithmType {
+ MD5(new Md5HashAlgorithm()), SHA1(new Sha1HashAlgorithm()), SHA2(new Sha2HashAlgorithm());
+
+ private HashAlgorithm hashAlgorithm;
+
+ HashAlgorithmType(HashAlgorithm hashAlgorithm) {
+ this.hashAlgorithm = hashAlgorithm;
+ }
+
+ public HashAlgorithm hashAlgorithm() {
+ return hashAlgorithm;
+ }
+}
diff --git a/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/Md5HashAlgorithm.java b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/Md5HashAlgorithm.java
new file mode 100644
index 0000000..6aafb86
--- /dev/null
+++ b/streampipes-processors-transformation-flink/src/main/java/org/streampipes/processors/transformation/flink/processor/hasher/algorithm/Md5HashAlgorithm.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018 FZI Forschungszentrum Informatik
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.streampipes.processors.transformation.flink.processor.hasher.algorithm;
+
+import org.apache.commons.codec.digest.DigestUtils;
+
+public class Md5HashAlgorithm implements HashAlgorithm {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public String toHashValue(Object value) {
+ return DigestUtils.md5Hex(String.valueOf(value));
+ }
+
+}
<
