Deploy Kafka with AMQ Streams and rook-ceph’s RBD Storage Class

Prerequisites

  • A running Openshift cluster (version 4.3.8)
  • A rook-ceph RBD storage class (In case you haven’t deployed reach out to my previous blog)
oc new-project amq-streams
Search for AMQ-Streams, and then click the install button
oc get podsamq-streams-cluster-operator-v1.4.0-f6d65d8b5-hqmrp   1/1     Running   1          43h
oc get scNAME                        PROVISIONER                  AGE
rook-ceph-block (default) rook-ceph.rbd.csi.ceph.com 43h
oc create -f - <<EOF 
apiVersion: kafka.strimzi.io/v1beta1
kind: Kafka
metadata:
name: kafka-cluster
namespace: amq-streams
labels:
app: amq-streams-demo
spec:
kafka:
version: 2.4.0
replicas: 3
listeners:
plain: {}
tls: {}
readinessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
livenessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
config:
offsets.topic.replication.factor: 3
transaction.state.log.replication.factor: 3
transaction.state.log.min.isr: 2
log.message.format.version: '2.2'
storage:
type: persistent-claim
size: 5Gi
deleteClaim: true
zookeeper:
replicas: 3
readinessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
livenessProbe:
initialDelaySeconds: 15
timeoutSeconds: 5
storage:
type: persistent-claim
size: 3Gi
deleteClaim: true
entityOperator:
topicOperator: {}
userOperator: {}
EOF
oc get pods  

NAME READY STATUS RESTARTS AGE
amq-streams-cluster-operator-v1.4.0-f6d65d8b5-vqn8n 1/1 Running 0 3m38s
kafka-cluster-entity-operator-5888fdb467-5sh2h 3/3 Running 0 58s
kafka-cluster-kafka-0 2/2 Running 0 111s
kafka-cluster-kafka-1 2/2 Running 0 111s
kafka-cluster-kafka-2 2/2 Running 0 111s
kafka-cluster-zookeeper-0 2/2 Running 0 2m43s
kafka-cluster-zookeeper-1 2/2 Running 0 2m43s
kafka-cluster-zookeeper-2 2/2 Running 0 2m43s
oc get pvpvc-378aee04-38f4-4e5f-a851-7e6c150b5057   5Gi        RWO            Delete           Bound       amq-streams/data-kafka-cluster-kafka-0                rook-ceph-block            4m34s
pvc-3ebc5c72-494e-48b5-b816-a6ac98d72c53 3Gi RWO Delete Bound amq-streams/data-kafka-cluster-zookeeper-0 rook-ceph-block 5m27s
pvc-4491a189-4cde-4dbb-8604-80cf0424b008 3Gi RWO Delete Bound amq-streams/data-kafka-cluster-zookeeper-1 rook-ceph-block 5m27s
pvc-779a7cb8-d1f1-4407-ad83-a85d6fe687c5 3Gi RWO Delete Bound amq-streams/data-kafka-cluster-zookeeper-2 rook-ceph-block 5m27s
pvc-a6eff9da-cba0-4e8b-92af-462b4c389bcc 5Gi RWO Delete Bound amq-streams/data-kafka-cluster-kafka-2 rook-ceph-block 4m34s
pvc-fab0f3bc-12cf-48d4-b614-3f8b9059d6e2 5Gi RWO Delete Bound amq-streams/data-kafka-cluster-kafka-1 rook-ceph-block 4m34s
oc create -f - <<EOF 
apiVersion: kafka.strimzi.io/v1beta1
kind: KafkaTopic
metadata:
name: my-topic
labels:
strimzi.io/cluster: kafka-cluster
namespace: amq-streams
spec:
partitions: 12
replicas: 3
config:
retention.ms: 604800000
segment.bytes: 1073741824
EOF
oc get kafkatopicNAME                                                         PARTITIONS   REPLICATION FACTOR
my-topic 12 3
oc create -f - <<EOF
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
app: hello-world-producer
name: hello-world-producer
spec:
replicas: 1
template:
metadata:
labels:
app: hello-world-producer
spec:
containers:
- name: hello-world-producer
image: strimzici/hello-world-producer:support-training
env:
- name: BOOTSTRAP_SERVERS
value: kafka-cluster-kafka-bootstrap:9092
- name: TOPIC
value: my-topic
- name: DELAY_MS
value: "1000"
- name: LOG_LEVEL
value: "INFO"
- name: MESSAGE_COUNT
value: "1000000"
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
labels:
app: hello-world-consumer
name: hello-world-consumer
spec:
replicas: 1
template:
metadata:
labels:
app: hello-world-consumer
spec:
containers:
- name: hello-world-consumer
image: strimzici/hello-world-consumer:support-training
env:
- name: BOOTSTRAP_SERVERS
value: kafka-cluster-kafka-bootstrap:9092
- name: TOPIC
value: my-topic
- name: GROUP_ID
value: my-hello-world-consumer
- name: LOG_LEVEL
value: "INFO"
- name: MESSAGE_COUNT
value: "1000000"
EOF
oc logs hello-world-producer-5d96c5b9df-547p92020-04-21 08:48:10 INFO  KafkaProducerExample:18 - Sending messages "Hello world - 1"
2020-04-21 08:48:11 INFO KafkaProducerExample:18 - Sending messages "Hello world - 2"
2020-04-21 08:48:12 INFO KafkaProducerExample:18 - Sending messages "Hello world - 3"
2020-04-21 08:48:13 INFO KafkaProducerExample:18 - Sending messages "Hello world - 4"
2020-04-21 08:48:14 INFO KafkaProducerExample:18 - Sending messages "Hello world - 5"
oc logs hello-world-consumer-8f9cd7dfd-lc7p42020-04-21 08:48:12 INFO  KafkaConsumerExample:25 - 	partition: 2
2020-04-21 08:48:12 INFO KafkaConsumerExample:26 - offset: 0
2020-04-21 08:48:12 INFO KafkaConsumerExample:27 - value: Hello world - 0
2020-04-21 08:48:12 INFO KafkaConsumerExample:24 - Received message:
2020-04-21 08:48:12 INFO KafkaConsumerExample:25 - partition: 9
2020-04-21 08:48:12 INFO KafkaConsumerExample:26 - offset: 0
2020-04-21 08:48:12 INFO KafkaConsumerExample:27 - value: Hello world - 2
2020-04-21 08:48:12 INFO KafkaConsumerExample:24 - Received message:
2020-04-21 08:48:12 INFO KafkaConsumerExample:25 - partition: 11
2020-04-21 08:48:12 INFO KafkaConsumerExample:26 - offset: 0
2020-04-21 08:48:12 INFO KafkaConsumerExample:27 - value: Hello world - 3
2020-04-21 08:48:13 INFO KafkaConsumerExample:24 - Received message:
2020-04-21 08:48:13 INFO KafkaConsumerExample:25 - partition: 4
2020-04-21 08:48:13 INFO KafkaConsumerExample:26 - offset: 0
2020-04-21 08:48:13 INFO KafkaConsumerExample:27 - value: Hello world - 4
2020-04-21 08:48:14 INFO KafkaConsumerExample:24 - Received message:
2020-04-21 08:48:14 INFO KafkaConsumerExample:25 - partition: 1
2020-04-21 08:48:14 INFO KafkaConsumerExample:26 - offset: 0
2020-04-21 08:48:14 INFO KafkaConsumerExample:27 - value: Hello world - 5
oc rsh kafka-cluster-kafka-0 bin/kafka-consumer-groups.sh --bootstrap-server kafka-cluster-kafka-bootstrap:9092 --describe --group my-hello-world-consumerGROUP                   TOPIC           PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG             CONSUMER-ID                                     HOST            CLIENT-ID
my-hello-world-consumer my-topic 2 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 11 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 1 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 6 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 5 25 25 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 10 25 25 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 9 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 7 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 0 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 4 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 8 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
my-hello-world-consumer my-topic 3 26 26 0 consumer-1-19d214cd-64a6-48bf-986f-bd696f11850f /10.128.0.73 consumer-1
oc scale deployment hello-world-consumer --replicas=0
oc rsh kafka-cluster-kafka-0 bin/kafka-consumer-groups.sh --bootstrap-server kafka-cluster-kafka-bootstrap:9092 --describe --group my-hello-world-consumerGROUP                   TOPIC           PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG             CONSUMER-ID     HOST            CLIENT-ID
my-hello-world-consumer my-topic 0 72 75 3 - - -
my-hello-world-consumer my-topic 10 72 75 3 - - -
my-hello-world-consumer my-topic 11 72 76 4 - - -
my-hello-world-consumer my-topic 1 72 76 4 - - -
my-hello-world-consumer my-topic 2 72 76 4 - - -
my-hello-world-consumer my-topic 3 72 75 3 - - -
my-hello-world-consumer my-topic 4 72 76 4 - - -
my-hello-world-consumer my-topic 5 72 75 3 - - -
my-hello-world-consumer my-topic 6 72 75 3 - - -
my-hello-world-consumer my-topic 7 72 75 3 - - -
my-hello-world-consumer my-topic 8 72 76 4 - - -
my-hello-world-consumer my-topic 9 72 76 4 - - -
oc rsh kafka-cluster-kafka-0 bin/kafka-consumer-groups.sh --bootstrap-server kafka-cluster-kafka-bootstrap:9092 --group my-hello-world-consumer --reset-offsets --topic my-topic --execute --to-earliestGROUP                          TOPIC                          PARTITION  NEW-OFFSET     
my-hello-world-consumer my-topic 2 0
my-hello-world-consumer my-topic 11 0
my-hello-world-consumer my-topic 1 0
my-hello-world-consumer my-topic 6 0
my-hello-world-consumer my-topic 5 0
my-hello-world-consumer my-topic 10 0
my-hello-world-consumer my-topic 9 0
my-hello-world-consumer my-topic 7 0
my-hello-world-consumer my-topic 0 0
my-hello-world-consumer my-topic 4 0
my-hello-world-consumer my-topic 8 0
my-hello-world-consumer my-topic 3 0
oc scale deployment hello-world-consumer --replicas=3
oc rsh kafka-cluster-kafka-0 bin/kafka-consumer-groups.sh --bootstrap-server kafka-cluster-kafka-bootstrap:9092 --describe --group my-hello-world-consumerGROUP                   TOPIC           PARTITION  CURRENT-OFFSET  LOG-END-OFFSET  LAG             CONSUMER-ID                                     HOST            CLIENT-ID
my-hello-world-consumer my-topic 0 110 110 0 consumer-1-138a3ce0-b3ad-431f-8876-c03da230e435 /10.128.0.75 consumer-1
my-hello-world-consumer my-topic 1 111 111 0 consumer-1-138a3ce0-b3ad-431f-8876-c03da230e435 /10.128.0.75 consumer-1
my-hello-world-consumer my-topic 2 111 111 0 consumer-1-138a3ce0-b3ad-431f-8876-c03da230e435 /10.128.0.75 consumer-1
my-hello-world-consumer my-topic 3 110 110 0 consumer-1-138a3ce0-b3ad-431f-8876-c03da230e435 /10.128.0.75 consumer-1
my-hello-world-consumer my-topic 8 111 111 0 consumer-1-74963626-2931-4502-af30-55cf96803dbc /10.128.0.77 consumer-1
my-hello-world-consumer my-topic 9 111 111 0 consumer-1-74963626-2931-4502-af30-55cf96803dbc /10.128.0.77 consumer-1
my-hello-world-consumer my-topic 10 110 110 0 consumer-1-74963626-2931-4502-af30-55cf96803dbc /10.128.0.77 consumer-1
my-hello-world-consumer my-topic 11 111 111 0 consumer-1-74963626-2931-4502-af30-55cf96803dbc /10.128.0.77 consumer-1
my-hello-world-consumer my-topic 4 111 111 0 consumer-1-6995a646-2af0-43c3-a9f5-3a386377ee1f /10.128.0.76 consumer-1
my-hello-world-consumer my-topic 5 110 110 0 consumer-1-6995a646-2af0-43c3-a9f5-3a386377ee1f /10.128.0.76 consumer-1
my-hello-world-consumer my-topic 6 110 110 0 consumer-1-6995a646-2af0-43c3-a9f5-3a386377ee1f /10.128.0.76 consumer-1
my-hello-world-consumer my-topic 7 110 110 0 consumer-1-6995a646-2af0-43c3-a9f5-3a386377ee1f /10.128.0.76 consumer-1
oc get pod hello-world-consumer-8f9cd7dfd-59xnd -o wideNAME                                   READY   STATUS    RESTARTS   AGE     IP            NODE                 NOMINATED NODE   READINESS GATES
hello-world-consumer-8f9cd7dfd-59xnd 1/1 Running 0 6m18s 10.128.0.77 crc-45nsk-master-0 <none> <none>
oc logs hello-world-consumer-8f9cd7dfd-59xnd2020-04-21 09:12:59 INFO  KafkaConsumerExample:24 - Received message:
2020-04-21 09:12:59 INFO KafkaConsumerExample:25 - partition: 10
2020-04-21 09:12:59 INFO KafkaConsumerExample:26 - offset: 123
2020-04-21 09:12:59 INFO KafkaConsumerExample:27 - value: Hello world - 1487
2020-04-21 09:13:01 INFO KafkaConsumerExample:24 - Received message:
2020-04-21 09:13:01 INFO KafkaConsumerExample:25 - partition: 8
2020-04-21 09:13:01 INFO KafkaConsumerExample:26 - offset: 124
2020-04-21 09:13:01 INFO KafkaConsumerExample:27 - value: Hello world - 1489
2020-04-21 09:13:02 INFO KafkaConsumerExample:24 - Received message:
2020-04-21 09:13:02 INFO KafkaConsumerExample:25 - partition: 9
2020-04-21 09:13:02 INFO KafkaConsumerExample:26 - offset: 124
2020-04-21 09:13:02 INFO KafkaConsumerExample:27 - value: Hello world - 1490

Conclusion

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Shon Paz

Shon Paz

Sr. Solution Architect, Red Hat