This is an automated email from the ASF dual-hosted git repository. kamir pushed a commit to branch kamir-patch-1 in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit d65f7cca17d91fa44d5601f07923b217b2987956 Merge: 1632e748 d05730f6 Author: Mirko Kämpf <[email protected]> AuthorDate: Wed Aug 14 18:19:56 2024 +0200 merged kafka source feature into kamir-patch-1-branch .gitignore | 9 + .move.sh | 69 ++++++ .../CardinalityEstimator.java => build/_env.sh | 29 +-- .../scala/org/apache/wayang/api/DataQuanta.scala | 38 +++ .../org/apache/wayang/api/DataQuantaBuilder.scala | 15 ++ .../org/apache/wayang/api/JavaPlanBuilder.scala | 11 +- wayang-commons/wayang-basic/pom.xml | 5 + .../wayang/basic/operators/KafkaTopicSink.java | 230 ++++++++++++++++++ .../wayang/basic/operators/KafkaTopicSource.java | 268 +++++++++++++++++++++ .../org/apache/wayang/core/api/Configuration.java | 3 +- .../wayang/core/function/FunctionDescriptor.java | 4 +- .../core/function/TransformationDescriptor.java | 11 +- .../optimizer/ProbabilisticDoubleInterval.java | 11 +- .../optimizer/ProbabilisticIntervalEstimate.java | 13 +- .../optimizer/cardinality/CardinalityEstimate.java | 6 +- .../cardinality/CardinalityEstimator.java | 4 +- .../cardinality/DefaultCardinalityEstimator.java | 13 +- .../wayang/core/optimizer/costs/LoadEstimate.java | 8 +- .../wayang/core/optimizer/costs/LoadProfile.java | 3 +- .../wayang/core/plan/wayangplan/InputSlot.java | 7 +- .../core/plan/wayangplan/OperatorAlternative.java | 9 +- .../wayang/core/plan/wayangplan/OperatorBase.java | 9 +- .../core/plan/wayangplan/OperatorContainer.java | 3 +- .../core/plan/wayangplan/OperatorContainers.java | 5 +- .../apache/wayang/core/plan/wayangplan/Slot.java | 11 +- .../wayang/core/plan/wayangplan/SlotMapping.java | 5 +- .../wayang/core/plan/wayangplan/UnarySink.java | 6 + .../wayang/core/types/BasicDataUnitType.java | 7 +- .../org/apache/wayang/core/types/DataSetType.java | 7 +- .../org/apache/wayang/core/types/DataUnitType.java | 5 +- ...che-Wayang-Architecture-Overview-MK-v01.graffle | Bin 0 -> 1621385 bytes .../Apache Wayang meets Apache Kafka.docx | Bin 0 -> 676382 bytes .../Scenario-cross-organizational-sharing.graffle | Bin 0 -> 153329 bytes wayang-platforms/wayang-java/pom.xml | 1 + .../wayang/java/mapping/KafkaTopicSinkMapping.java | 56 +++++ .../java/mapping/KafkaTopicSourceMapping.java | 56 +++++ .../org/apache/wayang/java/mapping/Mappings.java | 4 +- .../wayang/java/operators/JavaKafkaTopicSink.java | 149 ++++++++++++ .../java/operators/JavaKafkaTopicSource.java | 131 ++++++++++ .../main/resources/wayang-java-defaults.properties | 32 +++ .../resources/wayang-kafka-defaults.properties | 24 ++ .../java/operators/JavaKafkaTopicSourceTest.java | 141 +++++++++++ .../apache/wayang/java/test/KafkaClientTest.java | 134 +++++++++++ wayang-platforms/wayang-spark/pom.xml | 7 + .../spark/mapping/KafkaTopicSinkMapping.java | 56 +++++ .../spark/mapping/KafkaTopicSourceMapping.java | 55 +++++ .../org/apache/wayang/spark/mapping/Mappings.java | 5 +- .../spark/operators/SparkKafkaTopicSink.java | 133 ++++++++++ .../spark/operators/SparkKafkaTopicSource.java | 127 ++++++++++ 49 files changed, 1870 insertions(+), 65 deletions(-) diff --cc .gitignore index 533e6d3d,9b631067..b71b7a76 --- a/.gitignore +++ b/.gitignore @@@ -17,7 -20,8 +20,13 @@@ pom.xml. 10e*.input *_pb2.py + +# VSCode +.vscode +.vscode/* ++ + /build/env.sh + + /conf/kafka/default.properties + /wayang-platforms/wayang-java/src/main/resources/wayang-kafka-defaults.properties + /wayang-platforms/wayang-java/src/main/resources/wayang-kafka-defaults.properties.template diff --cc wayang-platforms/wayang-spark/pom.xml index 8b526ea7,3e3bf3dd..9b9b2843 --- a/wayang-platforms/wayang-spark/pom.xml +++ b/wayang-platforms/wayang-spark/pom.xml @@@ -76,32 -78,28 +76,39 @@@ <artifactId>kafka-clients</artifactId> <version>3.4.0</version> </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.12</artifactId> + <version>${spark.version}</version> + <exclusions> + <exclusion> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-graphx_2.12</artifactId> + <version>${spark.version}</version> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-mllib_2.12</artifactId> + <version>${spark.version}</version> + </dependency> + <!--Error of ArrayIndexOutOfBoundsException--> + <dependency> + <groupId>com.thoughtworks.paranamer</groupId> + <artifactId>paranamer</artifactId> + <version>2.8</version> + </dependency> + + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.15.4</version> + </dependency> + </dependencies> - - <modules> - <module>wayang-spark_2.12</module> - </modules> - - <profiles> - <profile> - <id>java8</id> - <activation> - <jdk>1.8</jdk> - </activation> - <modules> - <module>wayang-spark_2.11</module> - </modules> - </profile> - </profiles> </project> diff --cc wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/KafkaTopicSinkMapping.java index 00000000,4bae8e85..4bae8e85 mode 000000,100644..100644 --- a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/KafkaTopicSinkMapping.java +++ b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/KafkaTopicSinkMapping.java diff --cc wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/KafkaTopicSourceMapping.java index 00000000,220c6ca3..220c6ca3 mode 000000,100644..100644 --- a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/KafkaTopicSourceMapping.java +++ b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/KafkaTopicSourceMapping.java diff --cc wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkKafkaTopicSink.java index 00000000,0f6fe633..0f6fe633 mode 000000,100644..100644 --- a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkKafkaTopicSink.java +++ b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkKafkaTopicSink.java diff --cc wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkKafkaTopicSource.java index 00000000,a3831b32..a3831b32 mode 000000,100644..100644 --- a/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkKafkaTopicSource.java +++ b/wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkKafkaTopicSource.java
