[KARAF-5971] Add S3 appender
diff --git a/appender/pom.xml b/appender/pom.xml
index dcf4bad..35875a3 100644
--- a/appender/pom.xml
+++ b/appender/pom.xml
@@ -52,6 +52,7 @@
<module>prometheus</module>
<module>redis</module>
<module>rest</module>
+ <module>s3</module>
<module>socket</module>
<module>timescaledb</module>
<module>websocket-servlet</module>
diff --git a/appender/s3/NOTICE b/appender/s3/NOTICE
new file mode 100644
index 0000000..4e4af9e
--- /dev/null
+++ b/appender/s3/NOTICE
@@ -0,0 +1,57 @@
+Apache Karaf Decanter
+Copyright 2015-2019 The Apache Software Foundation
+
+I. Included Software
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+Licensed under the Apache License 2.0.
+
+This product includes software developed at
+Elastic (https://www.elastic.co/).
+Licensed under the Apache License 2.0.
+
+This product includes software developed at
+OrientDB (http://orientdb.com).
+Licensed under the Apache License 2.0.
+
+II. Used Software
+
+This product uses software developed at
+The OSGi Alliance (http://www.osgi.org/).
+Copyright (c) OSGi Alliance (2000, 2010).
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+OPS4J (http://www.ops4j.org/).
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+SLF4J (http://www.slf4j.org/).
+Licensed under the MIT License.
+
+This product uses software developed at
+JUnit (http://www.junit.org/).
+Licensed under the Eclipse Public License 1.0.
+
+This product uses software developed at
+Redis (http://www.redis.io).
+Licensed under the BSD license.
+
+This product uses software developed at
+Dropwizard (http://www.dropwizard.io).
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+searchbox.io (https://github.com/searchbox-io)
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+MongoDB (https://www.mongodb.com/)
+Licensed under the Apache License 2.0.
+
+III. License Summary
+- Apache License 2.0
+- MIT License
+- Eclipse Public License 1.0
+- BSD License
diff --git a/appender/s3/pom.xml b/appender/s3/pom.xml
new file mode 100644
index 0000000..50ece1a
--- /dev/null
+++ b/appender/s3/pom.xml
@@ -0,0 +1,146 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.karaf.decanter</groupId>
+ <artifactId>appender</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.karaf.decanter.appender</groupId>
+ <artifactId>org.apache.karaf.decanter.appender.s3</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Karaf :: Decanter :: Appender :: AWS S3</name>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bom</artifactId>
+ <version>1.11.837</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-sts</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-iam</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-kms</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.decanter</groupId>
+ <artifactId>org.apache.karaf.decanter.api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.decanter.appender</groupId>
+ <artifactId>org.apache.karaf.decanter.appender.utils</artifactId>
+ </dependency>
+
+ <!-- test -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.decanter.marshaller</groupId>
+ <artifactId>org.apache.karaf.decanter.marshaller.csv</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <inherited>true</inherited>
+ <extensions>true</extensions>
+ <configuration>
+ <obrRepository>NONE</obrRepository>
+ <instructions>
+ <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
+ <Export-Package>!*</Export-Package>
+ <Import-Package>
+ !com.amazonaws*,
+ !software.amazon*,
+ !org.apache.http*,
+ !org.joda*,
+ 8org.apache.commons.codec*,
+ com.sun*;resolution:=optional,
+ kotlin*;resolution:=optional,
+ org.bouncycastle*;resolution:=optional,
+ *
+ </Import-Package>
+ <Private-Package>
+ org.apache.karaf.decanter.appender.s3,
+ org.apache.karaf.decanter.appender.utils,
+ com.amazonaws*,
+ software.amazon*,
+ org.apache.http*,
+ org.apache.commons.codec*,
+ org.joda*
+ </Private-Package>
+ <_dsannotations>*</_dsannotations>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+ <file>src/main/cfg/org.apache.karaf.decanter.appender.s3.cfg</file>
+ <type>cfg</type>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/appender/s3/src/main/cfg/org.apache.karaf.decanter.appender.s3.cfg b/appender/s3/src/main/cfg/org.apache.karaf.decanter.appender.s3.cfg
new file mode 100644
index 0000000..efaea65
--- /dev/null
+++ b/appender/s3/src/main/cfg/org.apache.karaf.decanter.appender.s3.cfg
@@ -0,0 +1,36 @@
+################################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+###############################
+# Decanter Appender S3 Configuration
+###############################
+
+# AWS credentials
+accessKeyId=
+secretKeyId=
+
+# AWS Region (optional)
+#region=
+
+# S3 bucket name
+bucket=
+
+
+# Marshaller to use
+marshaller.target=(dataFormat=json)
diff --git a/appender/s3/src/main/java/org/apache/karaf/decanter/appender/s3/S3Appender.java b/appender/s3/src/main/java/org/apache/karaf/decanter/appender/s3/S3Appender.java
new file mode 100644
index 0000000..b2a41f7
--- /dev/null
+++ b/appender/s3/src/main/java/org/apache/karaf/decanter/appender/s3/S3Appender.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.karaf.decanter.appender.s3;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import org.apache.karaf.decanter.api.marshaller.Marshaller;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.event.Event;
+import org.osgi.service.event.EventConstants;
+import org.osgi.service.event.EventHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Dictionary;
+import java.util.UUID;
+
+@Component(
+ name = "org.apache.karaf.decanter.appender.s3",
+ immediate = true,
+ property = EventConstants.EVENT_TOPIC + "=decanter/collect/*"
+)
+public class S3Appender implements EventHandler {
+
+ private final static Logger LOGGER = LoggerFactory.getLogger(S3Appender.class);
+
+ private Regions regions;
+ private String accessKeyId;
+ private String secretKeyId;
+ private String bucket;
+
+ @Activate
+ public void activate(ComponentContext componentContext) {
+ activate(componentContext.getProperties());
+ }
+
+ public void activate(Dictionary<String, Object> config) {
+ if (config.get("accessKeyId") == null) {
+ throw new IllegalStateException("accessKeyId is not set");
+ }
+ accessKeyId = (String) config.get("accessKeyId");
+ if (config.get("secretKeyId") == null) {
+ throw new IllegalStateException("secretKeyId is not set");
+ }
+ secretKeyId = (String) config.get("secretKeyId");
+ if (config.get("bucket") == null) {
+ throw new IllegalStateException("bucket is not set");
+ }
+ bucket = (String) config.get("bucket");
+ regions = (config.get("region") != null) ? Regions.fromName((String) config.get("region")) : Regions.DEFAULT_REGION;
+ }
+
+ @Override
+ public void handleEvent(Event event) {
+ BasicAWSCredentials awsCreds = new BasicAWSCredentials(accessKeyId, secretKeyId);
+ AmazonS3 s3 = AmazonS3ClientBuilder.standard()
+ .withRegion(regions)
+ .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
+ .build();
+ String key = "decanter-" + System.currentTimeMillis() + "-" + UUID.randomUUID().toString();
+ s3.putObject(bucket, key, marshaller.marshal(event));
+ }
+
+ @Reference
+ public Marshaller marshaller;
+
+}
diff --git a/assembly/src/main/feature/feature.xml b/assembly/src/main/feature/feature.xml
index 2b489f0..74d84c5 100644
--- a/assembly/src/main/feature/feature.xml
+++ b/assembly/src/main/feature/feature.xml
@@ -393,6 +393,16 @@
<bundle>mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.hdfs/${project.version}</bundle>
</feature>
+ <feature name="decanter-appender-s3" version="${project.version}" description="Karaf Decanter AWS S3 appender">
+ <feature>decanter-common</feature>
+ <configfile finalname="/etc/org.apache.karaf.decanter.appender.s3.cfg">mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.s3/${project.version}</configfile>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/2.12.0</bundle>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-databind/2.12.0</bundle>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-annotations/2.12.0</bundle>
+ <bundle dependency="true">mvn:com.fasterxml.jackson.dataformat/jackson-dataformat-cbor/2.12.0</bundle>
+ <bundle>mvn:org.apache.karaf.decanter.appender/org.apache.karaf.decanter.appender.s3/${project.version}</bundle>
+ </feature>
+
<feature name="decanter-appender-redis-core" version="${project.version}" description="Karaf Decanter Redis Appender core">
<feature>decanter-common</feature>
<bundle dependency="true">mvn:com.fasterxml.jackson.core/jackson-core/2.12.0</bundle>
diff --git a/manual/src/main/asciidoc/user-guide/appenders.adoc b/manual/src/main/asciidoc/user-guide/appenders.adoc
index 72f7b1f..b29f96a 100644
--- a/manual/src/main/asciidoc/user-guide/appenders.adoc
+++ b/manual/src/main/asciidoc/user-guide/appenders.adoc
@@ -897,4 +897,41 @@
* `hdfs.configuration` is the location of the hdfs configuration file (core or site)
* `hdfs.mode` defines the way of populating the file on HDFS (creating a new one, appending to an existing one, overwriting an existing one)
-* `hdfs.path` defines the location and name of the file on HDFS
\ No newline at end of file
+* `hdfs.path` defines the location and name of the file on HDFS
+
+==== Amazon S3
+
+Decanter Amazon S3 appender stores collected data as objects in a S3 bucket.
+
+The `decanter-appender-s3` feature installs the S3 appender:
+
+----
+karaf@root()> feature:install decanter-appender-s3
+----
+
+The feature also installs `etc/org.apache.karaf.decanter.appender.hdfs.cfg` configuration file:
+
+----
+###############################
+# Decanter Appender S3 Configuration
+###############################
+
+# AWS credentials
+accessKeyId=
+secretKeyId=
+
+# AWS Region (optional)
+#region=
+
+# S3 bucket name
+bucket=
+
+
+# Marshaller to use
+marshaller.target=(dataFormat=json)
+----
+
+* `accessKeyId` property is required, containing your AWS access key
+* `secretKeyId` property is required, containing your AWS secret key
+* `region` property is optional and allows you to define the Amazon region to use
+* `bucket` property is required, containing the name of the S3 bucket where to add objects
\ No newline at end of file
