From 17dc5c0dd3baa5ef91c5fd5ebe59693597acd3dc Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Wed, 8 Oct 2025 18:17:15 +0000 Subject: [PATCH 1/4] switch to apache kafka --- .../flex-templates/kafka_to_bigquery/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dataflow/flex-templates/kafka_to_bigquery/README.md b/dataflow/flex-templates/kafka_to_bigquery/README.md index 4c8121fc95c..b54741ec849 100644 --- a/dataflow/flex-templates/kafka_to_bigquery/README.md +++ b/dataflow/flex-templates/kafka_to_bigquery/README.md @@ -234,8 +234,8 @@ $ gcloud compute ssh kafka-vm --zone "$ZONE" 1. Create a Topic ```sh -docker run --rm --network host bitnami/kafka:3.4.0 \ -/opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 \ +docker run --rm --network host apache/kafka:latest \ +/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 \ --create --topic --partitions 1 --replication-factor 1 ``` @@ -250,8 +250,8 @@ Note: You can run this step either before starting the Dataflow template they arrive). ```sh -docker run -i --rm --network host bitnami/kafka:3.4.0 \ -/opt/bitnami/kafka/bin/kafka-console-producer.sh \ +docker run -i --rm --network host apache/kafka:latest \ +/opt/kafka/bin/kafka-console-producer.sh \ --bootstrap-server localhost:9092 --topic ``` @@ -262,16 +262,16 @@ This will print all messages from the beginning of the topic. Press Ctrl+C to exit. ```sh -docker run -it --rm --network host bitnami/kafka:3.4.0 \ -/opt/bitnami/kafka/bin/kafka-console-consumer.sh \ +docker run -it --rm --network host apache/kafka:latest \ +/opt/kafka/bin/kafka-console-consumer.sh \ --bootstrap-server localhost:9092 --topic --from-beginning ``` 4. (Optional) Delete a Topic ```sh -docker run --rm --network host bitnami/kafka:3.4.0 \ -/opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 \ +docker run --rm --network host apache/kafka:latest \ +/opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 \ --delete --topic ``` From 12ca20cc1f50439568608cd7a40b3cb4594ff1b6 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 30 Dec 2025 17:57:45 +0000 Subject: [PATCH 2/4] updated per comments --- .../flex-templates/kafka_to_bigquery/README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dataflow/flex-templates/kafka_to_bigquery/README.md b/dataflow/flex-templates/kafka_to_bigquery/README.md index b54741ec849..fe79f0068fd 100644 --- a/dataflow/flex-templates/kafka_to_bigquery/README.md +++ b/dataflow/flex-templates/kafka_to_bigquery/README.md @@ -8,6 +8,12 @@ Samples showing how to create and run an ## Before you begin +NOTE: These instructions used to use Bitnami version of Kafka, but due to a recent +untagging event on [Docker Hub]((https://hub.docker.com/r/bitnami/kafka)) for +all Bitnami images, these instructions have switched over to Apache Kafka. +They are still pinned to a Zookeeper compatible [version](https://kafka.apache.org/blog/2024/11/06/apache-kafka-3.9.0-release-announcement/) +of Kafka for now. + If you are not familiar with Dataflow Flex templates, please see the [Streaming Beam SQL](../streaming-beam-sql/) sample first. @@ -234,7 +240,7 @@ $ gcloud compute ssh kafka-vm --zone "$ZONE" 1. Create a Topic ```sh -docker run --rm --network host apache/kafka:latest \ +docker run --rm --network host apache/kafka:3.9.1 \ /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 \ --create --topic --partitions 1 --replication-factor 1 ``` @@ -250,7 +256,7 @@ Note: You can run this step either before starting the Dataflow template they arrive). ```sh -docker run -i --rm --network host apache/kafka:latest \ +docker run -i --rm --network host apache/kafka:3.9.1 \ /opt/kafka/bin/kafka-console-producer.sh \ --bootstrap-server localhost:9092 --topic ``` @@ -262,7 +268,7 @@ This will print all messages from the beginning of the topic. Press Ctrl+C to exit. ```sh -docker run -it --rm --network host apache/kafka:latest \ +docker run -it --rm --network host apache/kafka:3.9.1 \ /opt/kafka/bin/kafka-console-consumer.sh \ --bootstrap-server localhost:9092 --topic --from-beginning ``` @@ -270,7 +276,7 @@ docker run -it --rm --network host apache/kafka:latest \ 4. (Optional) Delete a Topic ```sh -docker run --rm --network host apache/kafka:latest \ +docker run --rm --network host apache/kafka:3.9.1 \ /opt/kafka/bin/kafka-topics.sh --bootstrap-server localhost:9092 \ --delete --topic ``` From 78164dcde258223eb255d98364d04ddf4a89f590 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 30 Dec 2025 17:58:52 +0000 Subject: [PATCH 3/4] minor fix --- dataflow/flex-templates/kafka_to_bigquery/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dataflow/flex-templates/kafka_to_bigquery/README.md b/dataflow/flex-templates/kafka_to_bigquery/README.md index fe79f0068fd..720eedb13b3 100644 --- a/dataflow/flex-templates/kafka_to_bigquery/README.md +++ b/dataflow/flex-templates/kafka_to_bigquery/README.md @@ -10,8 +10,9 @@ Samples showing how to create and run an NOTE: These instructions used to use Bitnami version of Kafka, but due to a recent untagging event on [Docker Hub]((https://hub.docker.com/r/bitnami/kafka)) for -all Bitnami images, these instructions have switched over to Apache Kafka. -They are still pinned to a Zookeeper compatible [version](https://kafka.apache.org/blog/2024/11/06/apache-kafka-3.9.0-release-announcement/) +all Bitnami images, these instructions have switched over to Apache Kafka images. +They are still pinned to a Zookeeper compatible +[version](https://kafka.apache.org/blog/2024/11/06/apache-kafka-3.9.0-release-announcement/) of Kafka for now. If you are not familiar with Dataflow Flex templates, please see the From ab343726c5a2ecec21114f69aa211a793408e098 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 30 Dec 2025 17:59:34 +0000 Subject: [PATCH 4/4] fix double (()) --- dataflow/flex-templates/kafka_to_bigquery/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataflow/flex-templates/kafka_to_bigquery/README.md b/dataflow/flex-templates/kafka_to_bigquery/README.md index 720eedb13b3..af795228a22 100644 --- a/dataflow/flex-templates/kafka_to_bigquery/README.md +++ b/dataflow/flex-templates/kafka_to_bigquery/README.md @@ -9,7 +9,7 @@ Samples showing how to create and run an ## Before you begin NOTE: These instructions used to use Bitnami version of Kafka, but due to a recent -untagging event on [Docker Hub]((https://hub.docker.com/r/bitnami/kafka)) for +untagging event on [Docker Hub](https://hub.docker.com/r/bitnami/kafka) for all Bitnami images, these instructions have switched over to Apache Kafka images. They are still pinned to a Zookeeper compatible [version](https://kafka.apache.org/blog/2024/11/06/apache-kafka-3.9.0-release-announcement/)