CAMEL-20914 - Camel-AWS-Kinesis: KCL Consumers add an example (#140)

Signed-off-by: Andrea Cosentino <[email protected]>

README.adoc

@@ -28,7 +28,7 @@ readme's instructions.
 == Examples
 
 // examples: START
-Number of Examples: 76 (0 deprecated)
+Number of Examples: 77 (0 deprecated)
 
 [width="100%",cols="4,2,4",options="header"]
 |===
@@ -90,6 +90,8 @@ Number of Examples: 76 (0 deprecated)
 
 | link:main-endpointdsl/README.adoc[Main Endpointdsl] (main-endpointdsl) | Beginner | An example for showing standalone Camel with Endpoint DSL
 
+| link:aws/main-endpointdsl-aws-kcl-kinesis/README.adoc[Main Endpointdsl Aws Kcl Kinesis] (main-endpointdsl-aws-kcl-kinesis) | Beginner | An example for showing standalone Camel with Endpoint DSL and AWS2-Kinesis
+
 | link:aws/main-endpointdsl-aws2-s3/README.adoc[Main Endpointdsl Aws2 S3] (main-endpointdsl-aws2-s3) | Beginner | An example for showing standalone Camel with Endpoint DSL and AWS2-S3
 
 | link:aws/main-endpointdsl-aws2-s3-kafka/README.adoc[Main Endpointdsl Aws2 S3 Kafka] (main-endpointdsl-aws2-s3-kafka) | Beginner | An example for showing standalone Camel with Endpoint DSL and AWS2-S3 and Kafka

aws/main-endpointdsl-aws-kcl-kinesis/README.adoc

@@ -0,0 +1,43 @@
+== Camel Example Main Endpoint DSL with AWS2 Kinesis component
+
+This example shows how to use the endpoint DSL in your Camel routes
+to define endpoints using type safe fluent builders, which are Java methods
+that are compiled.
+
+The example will poll a Kinesis Data Stream by using KCL consumers.
+
+You'll need to create the stream from your AWS console or your AWS CLI.
+
+Notice how you can configure Camel in the `application.properties` file.
+
+This example will use the AWS default credentials Provider: https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
+Set your credentials accordingly.
+Don't forget to add the stream name.
+
+=== Build
+
+First compile the example by executing:
+
+[source,sh]
+----
+$ mvn compile
+----
+
+=== How to run
+
+You can run this example using
+
+[source,sh]
+----
+$ mvn camel:run
+----
+
+=== Help and contributions
+
+If you hit any problem using Camel or have some feedback, then please
+https://camel.apache.org/community/support/[let us know].
+
+We also love contributors, so
+https://camel.apache.org/community/contributing/[get involved] :-)
+
+The Camel riders!

aws/main-endpointdsl-aws-kcl-kinesis/pom.xml

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.apache.camel.example</groupId>
+        <artifactId>camel-examples-aws-parent</artifactId>
+        <version>4.7.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>camel-example-main-endpointdsl-aws2-kinesis-kcl</artifactId>
+    <packaging>jar</packaging>
+    <name>Camel :: Example :: Main :: Endpoint DSL :: AWS2 KCL Kinesis</name>
+    <description>An example for showing standalone Camel with Endpoint DSL and AWS2-Kinesis</description>
+
+    <properties>
+        <category>Beginner</category>
+    </properties>
+
+    <dependencyManagement>
+        <dependencies>
+            <!-- Add Camel BOM -->
+            <dependency>
+                <groupId>org.apache.camel</groupId>
+                <artifactId>camel-bom</artifactId>
+                <version>${camel.version}</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-main</artifactId>
+        </dependency>
+        <!-- we use the endpoint-dsl -->
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-endpointdsl</artifactId>
+        </dependency>
+        <!-- we use these 2 camel components in this example -->
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-bean</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-aws2-kinesis</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-timer</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.camel</groupId>
+            <artifactId>camel-file</artifactId>
+        </dependency>
+
+        <!-- logging -->
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-core</artifactId>
+            <version>${log4j2-version}</version>
+            <scope>runtime</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.logging.log4j</groupId>
+            <artifactId>log4j-slf4j2-impl</artifactId>
+            <version>${log4j2-version}</version>
+            <scope>runtime</scope>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <!-- to run the application -->
+            <plugin>
+                <groupId>org.apache.camel</groupId>
+                <artifactId>camel-maven-plugin</artifactId>
+                <version>${camel.version}</version>
+                <configuration>
+                    <mainClass>org.apache.camel.example.MyApplication</mainClass>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyApplication.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.example;
+
+import org.apache.camel.main.Main;
+
+/**
+ * Main class that boot the Camel application
+ */
+public final class MyApplication {
+
+    private MyApplication() {
+    }
+
+    public static void main(String[] args) throws Exception {
+        // use Camels Main class
+        Main main = new Main(MyApplication.class);
+        // now keep the application running until the JVM is terminated (ctrl + c or sigterm)
+        main.run(args);
+    }
+
+}

aws/main-endpointdsl-aws-kcl-kinesis/src/main/java/org/apache/camel/example/MyRouteBuilder.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.example;
+
+import org.apache.camel.Exchange;
+import org.apache.camel.Processor;
+import org.apache.camel.builder.endpoint.EndpointRouteBuilder;
+import org.apache.camel.component.aws2.kinesis.Kinesis2Constants;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * To use the endpoint DSL then we must extend EndpointRouteBuilder instead of RouteBuilder
+ */
+public class MyRouteBuilder extends EndpointRouteBuilder {
+
+    @Override
+    public void configure() throws Exception {
+
+        from(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true).advanced().useKclConsumers(true).asyncClient(true))
+            .log("The content is ${body} from ${headers.CamelAwsKinesisShardId}").process(new Processor() {
+                    @Override
+                    public void process(Exchange exchange) throws Exception {
+                        ByteBuffer buffer = exchange.getMessage().getBody(ByteBuffer.class);
+                        buffer.flip();
+                        CharBuffer charBuffer = StandardCharsets.UTF_8.decode(buffer);
+                        exchange.getMessage().setBody(charBuffer.toString());
+                    }
+                }).to(file("./target/kcl-files/")).startupOrder(1);
+
+        from(timer("kcl-ingest-1").repeatCount(5))
+                .setBody(constant("Camel KCL Test Partition 1"))
+                .setHeader(Kinesis2Constants.PARTITION_KEY, constant("partition-1"))
+                .to(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true)).startupOrder(2);
+
+        from(timer("kcl-ingest-2").repeatCount(5))
+                .setBody(constant("Camel KCL Test Partition 2"))
+                .setHeader(Kinesis2Constants.PARTITION_KEY, constant("partition-2"))
+                .to(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true)).startupOrder(3);
+
+        from(timer("kcl-ingest-3").repeatCount(5))
+                .setBody(constant("Camel KCL Test Partition 3"))
+                .setHeader(Kinesis2Constants.PARTITION_KEY, constant("partition-3"))
+                .to(aws2Kinesis("{{streamName}}").useDefaultCredentialsProvider(true)).startupOrder(4);
+    }
+}

aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/application.properties

@@ -0,0 +1,23 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements.  See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License.  You may obtain a copy of the License at
+##
+##      http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+# here you can configure options on camel main
+# https://camel.apache.org/components/next/others/main.html
+camel.main.name = AWS2-KCL-Kinesis
+
+# properties used in the route
+streamName=kcl-stream

aws/main-endpointdsl-aws-kcl-kinesis/src/main/resources/log4j2.properties

@@ -0,0 +1,23 @@
+## ---------------------------------------------------------------------------
+## Licensed to the Apache Software Foundation (ASF) under one or more
+## contributor license agreements.  See the NOTICE file distributed with
+## this work for additional information regarding copyright ownership.
+## The ASF licenses this file to You under the Apache License, Version 2.0
+## (the "License"); you may not use this file except in compliance with
+## the License.  You may obtain a copy of the License at
+##
+##      http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+## ---------------------------------------------------------------------------
+
+appender.out.type = Console
+appender.out.name = out
+appender.out.layout.type = PatternLayout
+appender.out.layout.pattern = [%30.30t] %-30.30c{1} %-5p %m%n
+rootLogger.level = INFO
+rootLogger.appenderRef.out.ref = out

aws/pom.xml

@@ -40,6 +40,7 @@
         <module>main-endpointdsl-aws2</module>
         <module>main-endpointdsl-aws2-s3</module>
         <module>main-endpointdsl-aws2-s3-kafka</module>
+        <module>main-endpointdsl-aws-kcl-kinesis</module>
         <module>main-endpointdsl-kafka-aws2-s3</module>
         <module>main-endpointdsl-kafka-aws2-s3-restarting-policy</module>
     </modules>

main-tiny/src/generated/java/org/apache/camel/example/MyBeanConfigurer.java

@@ -1,6 +1,7 @@
 /* Generated by camel build tools - do NOT edit this file! */
 package org.apache.camel.example;
 
+import javax.annotation.processing.Generated;
 import java.util.Map;
 
 import org.apache.camel.CamelContext;
@@ -14,6 +15,7 @@
 /**
  * Generated by camel build tools - do NOT edit this file!
  */
+@Generated("org.apache.camel.maven.packaging.GenerateConfigurerMojo")
 @SuppressWarnings("unchecked")
 public class MyBeanConfigurer extends org.apache.camel.support.component.PropertyConfigurerSupport implements GeneratedPropertyConfigurer, PropertyConfigurerGetter {