Updating kafka version and fixing node-exporter jobs in prometheus configuration file

Author: daniel-gonzalez-sanchez

docker-compose-stream-processing.yml

@@ -45,7 +45,7 @@ services:
         jobmanager.rpc.address: flink-jobmanager
         taskmanager.numberOfTaskSlots: 2
   kafka:
-    image: wurstmeister/kafka
+    image: wurstmeister/kafka:latest
     hostname: kafka
     container_name: kafka
     ports:
@@ -81,7 +81,7 @@ services:
       - kafka
       - zookeeper
     volumes:
-      - ./kafka/streams/streams.examples:/kafka_2.13-2.6.0/streams.examples
+      - ./kafka/streams/streams.examples:/kafka_2.13-2.7.0/streams.examples
     command: tail -f /dev/null
   node-exporter-1:
     image: prom/node-exporter

kafka/connect/Dockerfile

@@ -1,18 +1,18 @@
-FROM wurstmeister/kafka
+FROM wurstmeister/kafka:latest
 
 # plugins folder
-RUN mkdir /opt/kafka_2.13-2.6.0/plugins
+RUN mkdir /opt/kafka_2.13-2.7.0/plugins
 
-# kafka-connect-http plugin
-COPY ./config/plugins /opt/kafka_2.13-2.6.0/plugins/
+# kafka-connect-http plugins
+COPY ./config/plugins /opt/kafka_2.13-2.7.0/plugins/
 
 # we replace the default connect-distributed.properties so we can properly resolve to our local kafka docker development
-COPY ./config/connect-distributed.properties /opt/kafka_2.13-2.6.0/config/
+COPY ./config/connect-distributed.properties /opt/kafka_2.13-2.7.0/config/
 
 # we replace the start command creating a connector instead.
 COPY ./config/start-kafka.sh /usr/bin/
 
 # permissions
 RUN chmod a+x /usr/bin/start-kafka.sh
 
-WORKDIR /opt/kafka_2.13-2.6.0
+WORKDIR /opt/kafka_2.13-2.7.0

kafka/connect/config/connect-distributed.properties

@@ -84,4 +84,4 @@ offset.flush.interval.ms=10000
 # Examples: 
 # plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
 #plugin.path=/usr/local/share/kafka/plugins
-plugin.path=/opt/kafka_2.13-2.6.0/plugins
+plugin.path=/opt/kafka_2.13-2.7.0/plugins

kafka/connect/config/start-kafka.sh

@@ -1,4 +1,4 @@
 #!/bin/bash -e
 
 # connector start command here.
-exec "/opt/kafka_2.13-2.6.0/bin/connect-distributed.sh" "/opt/kafka_2.13-2.6.0/config/connect-distributed.properties"
+exec "/opt/kafka_2.13-2.7.0/bin/connect-distributed.sh" "/opt/kafka_2.13-2.7.0/config/connect-distributed.properties"

kafka/streams/Dockerfile

@@ -6,11 +6,11 @@ RUN apt-get install -y openjdk-8-jdk nano wget curl
 
 RUN apt-get install -y maven
 
-RUN wget http://apache.uvigo.es/kafka/2.6.0/kafka_2.13-2.6.0.tgz && tar -zxvf kafka_2.13-2.6.0.tgz
+RUN wget http://apache.uvigo.es/kafka/2.7.0/kafka_2.13-2.7.0.tgz && tar -zxvf kafka_2.13-2.7.0.tgz
 
 # streams apps folder
-RUN mkdir kafka_2.13-2.6.0/streams.examples
+RUN mkdir kafka_2.13-2.7.0/streams.examples
 
-COPY ./streams.examples kafka_2.13-2.6.0/streams.examples
+COPY ./streams.examples kafka_2.13-2.7.0/streams.examples
 
-WORKDIR kafka_2.13-2.6.0
+WORKDIR kafka_2.13-2.7.0

kafka/streams/streams.examples/target/classes/java-json.jar

@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+log4j.rootLogger=INFO, console
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n

kafka/streams/streams.examples/target/classes/log4j.properties

@@ -0,0 +1,5 @@
+#Generated by Maven
+#Mon Jan 11 14:20:17 GMT 2021
+version=0.1
+groupId=streams.examples
+artifactId=streams.examples

kafka/streams/streams.examples/target/classes/myapps/CPULoad$1.class

@@ -0,0 +1,8 @@
+myapps/WordCount.class
+myapps/CPULoad$1.class
+myapps/LineSplit$1.class
+myapps/LineSplit.class
+myapps/WordCount$1.class
+myapps/CPULoad.class
+myapps/Pipe$1.class
+myapps/Pipe.class

kafka/streams/streams.examples/target/classes/myapps/CPULoad.class

@@ -0,0 +1,4 @@
+/kafka_2.13-2.7.0/streams.examples/src/main/java/myapps/CPULoad.java
+/kafka_2.13-2.7.0/streams.examples/src/main/java/myapps/LineSplit.java
+/kafka_2.13-2.7.0/streams.examples/src/main/java/myapps/WordCount.java
+/kafka_2.13-2.7.0/streams.examples/src/main/java/myapps/Pipe.java

kafka/streams/streams.examples/target/classes/myapps/LineSplit$1.class

@@ -26,12 +26,20 @@ scrape_configs:
     static_configs:
       - targets: ['localhost:9090']
 
-  - job_name: node
+  - job_name: 'node-1'
 
     scrape_interval: 5s
 
     static_configs:
-    - targets: ['node-exporter:9100']
+    - targets: ['node-exporter-1:9100']
+
+  - job_name: 'node-2'
+
+    scrape_interval: 5s
+
+    static_configs:
+    - targets: ['node-exporter-2:9100']
+ 
   # REQUIRES configuring docker daemon.json
   # - job_name: 'docker'
   #        # metrics_path defaults to '/metrics'