This is an automated email from the ASF dual-hosted git repository.
kamir pushed a commit to branch kamir-patch-2
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
The following commit(s) were added to refs/heads/kamir-patch-2 by this push:
new b2fee627 fixed license header.
b2fee627 is described below
commit b2fee627d4a95b2f0865ad0e166d13af95d34b88
Author: Mirko Kämpf <[email protected]>
AuthorDate: Tue Sep 10 17:22:54 2024 +0200
fixed license header.
---
.../bin/bootstrap_ccloud_kafka_topics.sh | 36 ++++++++--------------
.../bin/cleanup_ccloud_kafka_topics.sh | 15 +++++----
wayang-applications/bin/env.demo1.sh | 24 +++++++++++++++
wayang-applications/bin/env_template.sh | 22 +++++++++++--
wayang-applications/bin/run_wordcount.sh | 6 ++--
wayang-applications/bin/run_wordcount_kafka.sh | 9 +++---
6 files changed, 74 insertions(+), 38 deletions(-)
diff --git a/wayang-applications/bin/bootstrap_ccloud_kafka_topics.sh
b/wayang-applications/bin/bootstrap_ccloud_kafka_topics.sh
index b167c531..ff597af5 100755
--- a/wayang-applications/bin/bootstrap_ccloud_kafka_topics.sh
+++ b/wayang-applications/bin/bootstrap_ccloud_kafka_topics.sh
@@ -19,31 +19,21 @@
#brew install jq
#brew install git-lfs
-export topic_l1_a=region_emea_counts
-export topic_l1_b=region_apac_counts
-export topic_l1_c=region_uswest_counts
-export topic_l2_a=global_contribution
-export topic_l2_b=global_averages
+source .env.sh
+source env.demo1.sh
-confluent kafka topic delete topic_l1_a --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l1_b --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l1_c --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l2_a --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l2_b --cluster lkc-m2kpj2
+confluent kafka topic delete $topic_l1_a --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l1_b --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l1_c --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l2_a --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l2_b --cluster $DEMO1_CLUSTER1
-confluent kafka topic create topic_l1_a --cluster lkc-m2kpj2
-confluent kafka topic create topic_l1_b --cluster lkc-m2kpj2
-confluent kafka topic create topic_l1_c --cluster lkc-m2kpj2
-confluent kafka topic create topic_l2_a --cluster lkc-m2kpj2
-confluent kafka topic create topic_l2_b --cluster lkc-m2kpj2
+confluent kafka topic create $topic_l1_a --cluster $DEMO1_CLUSTER1
+confluent kafka topic create $topic_l1_b --cluster $DEMO1_CLUSTER1
+confluent kafka topic create $topic_l1_c --cluster $DEMO1_CLUSTER1
+confluent kafka topic create $topic_l2_a --cluster $DEMO1_CLUSTER1
+confluent kafka topic create $topic_l2_b --cluster $DEMO1_CLUSTER1
-######################################################################################################
-#
https://docs.confluent.io/cloud/current/sr/schema_registry_ccloud_tutorial.html
-export SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO="...:...."
-export SCHEMA_REGISTRY_URL="https://....";
+confluent kafka topic list --cluster $DEMO1_CLUSTER1
-confluent kafka topic list --cluster lkc-m2kpj2
-
-###
-# List schemas
curl --silent -X GET -u $SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO
$SCHEMA_REGISTRY_URL/subjects | jq .
diff --git a/wayang-applications/bin/cleanup_ccloud_kafka_topics.sh
b/wayang-applications/bin/cleanup_ccloud_kafka_topics.sh
index d3f8942f..8f78b4d0 100755
--- a/wayang-applications/bin/cleanup_ccloud_kafka_topics.sh
+++ b/wayang-applications/bin/cleanup_ccloud_kafka_topics.sh
@@ -15,10 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-confluent kafka topic delete topic_l1_a --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l1_b --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l1_c --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l2_a --cluster lkc-m2kpj2
-confluent kafka topic delete topic_l2_b --cluster lkc-m2kpj2
+source .env.sh
+source env.demo1.sh
-confluent kafka topic list --cluster lkc-m2kpj2
\ No newline at end of file
+confluent kafka topic delete $topic_l1_a --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l1_b --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l1_c --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l2_a --cluster $DEMO1_CLUSTER1
+confluent kafka topic delete $topic_l2_b --cluster $DEMO1_CLUSTER1
+
+confluent kafka topic list --cluster $DEMO1_CLUSTER1
\ No newline at end of file
diff --git a/wayang-applications/bin/env.demo1.sh
b/wayang-applications/bin/env.demo1.sh
index e69de29b..e89e9cc5 100644
--- a/wayang-applications/bin/env.demo1.sh
+++ b/wayang-applications/bin/env.demo1.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# topics for demo1
+export topic_l1_a=region_emea_counts
+export topic_l1_b=region_apac_counts
+export topic_l1_c=region_uswest_counts
+export topic_l2_a=global_contribution
+export topic_l2_b=global_averages
\ No newline at end of file
diff --git a/wayang-applications/bin/env_template.sh
b/wayang-applications/bin/env_template.sh
index 8fbcb253..1ad57688 100644
--- a/wayang-applications/bin/env_template.sh
+++ b/wayang-applications/bin/env_template.sh
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -17,4 +16,23 @@
# limitations under the License.
#
-export JAVA_HOME=....
\ No newline at end of file
+export JAVA_HOME=...
+export SPARK_HOME=...
+export HADOOP_HOME=...
+export PATH=$PATH:$HADOOP_HOME/bin
+export WAYANG_HOME=...
+export WAYANG_APP_HOME=...
+
+# properties of brokers and schema registry of Ccloud cluster for demo 1
+export BOOTSTRAP_SERVER=
+export CLUSTER_API_KEY=
+export CLUSTER_API_SECRET=
+export SR_ENDPOINT=
+export SR_API_KEY=
+export SR_API_SECRET=
+
+export SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO="...:..."
+export SCHEMA_REGISTRY_URL="..."
+
+# cluster-id of Ccloud cluster...
+export DEMO1_CLUSTER1=...
diff --git a/wayang-applications/bin/run_wordcount.sh
b/wayang-applications/bin/run_wordcount.sh
index 35c7544a..dbbd8ee3 100755
--- a/wayang-applications/bin/run_wordcount.sh
+++ b/wayang-applications/bin/run_wordcount.sh
@@ -21,14 +21,14 @@
#
source .env.sh
-cd ..
-cd ..
+cd ../..
mvn clean compile package install -pl :wayang-assembly -Pdistribution
-DskipTests
cd wayang-applications
+
mvn compile package install -DskipTests
cd ..
-source ./.env.sh; bin/wayang-submit org.apache.wayang.applications.WordCount
java file://$(pwd)/wayang-applications/data/case-study/DATA_REPO_001/README.md
+bin/wayang-submit org.apache.wayang.applications.WordCount java
file://$(pwd)/wayang-applications/data/case-study/DATA_REPO_001/README.md
diff --git a/wayang-applications/bin/run_wordcount_kafka.sh
b/wayang-applications/bin/run_wordcount_kafka.sh
index fcaafcd4..3c82338b 100755
--- a/wayang-applications/bin/run_wordcount_kafka.sh
+++ b/wayang-applications/bin/run_wordcount_kafka.sh
@@ -20,15 +20,16 @@
# to adjust variables to your own environment, please configure them in env.sh
#
source .env.sh
+source env.demo1.sh
-cd ..
-cd ..
+cd ../..
-#mvn clean compile package install -pl :wayang-assembly -Pdistribution
-DskipTests
+mvn clean compile package install -pl :wayang-assembly -Pdistribution
-DskipTests
cd wayang-applications
+
mvn compile package install -DskipTests
cd ..
-source ./.env.sh; bin/wayang-submit
org.apache.wayang.applications.WordCountOnKafkaTopic
\ No newline at end of file
+bin/wayang-submit org.apache.wayang.applications.WordCountOnKafkaTopic
\ No newline at end of file
