[2/2] spark git commit: Revert "[SPARK-22908] Add kafka source and sink for continuous processing."
Revert "[SPARK-22908] Add kafka source and sink for continuous processing."
This reverts commit f891ee3249e04576dd579cbab6f8f1632550e6bd.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/60bcb468
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/60bcb468
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/60bcb468
Branch: refs/heads/branch-2.3
Commit: 60bcb4685022c29a6ddcf707b505369687ec7da6
Parents: 02176f4
Author: Sameer Agarwal
Authored: Fri Jan 12 15:07:14 2018 -0800
Committer: Sameer Agarwal
Committed: Fri Jan 12 15:07:14 2018 -0800
--
.../sql/kafka010/KafkaContinuousReader.scala| 232 -
.../sql/kafka010/KafkaContinuousWriter.scala| 119 -
.../spark/sql/kafka010/KafkaOffsetReader.scala | 21 +-
.../apache/spark/sql/kafka010/KafkaSource.scala | 17 +-
.../spark/sql/kafka010/KafkaSourceOffset.scala | 7 +-
.../sql/kafka010/KafkaSourceProvider.scala | 105 +---
.../spark/sql/kafka010/KafkaWriteTask.scala | 71 ++-
.../apache/spark/sql/kafka010/KafkaWriter.scala | 5 +-
.../sql/kafka010/KafkaContinuousSinkSuite.scala | 474 ---
.../kafka010/KafkaContinuousSourceSuite.scala | 96
.../sql/kafka010/KafkaContinuousTest.scala | 64 ---
.../spark/sql/kafka010/KafkaSourceSuite.scala | 470 +-
.../org/apache/spark/sql/DataFrameReader.scala | 32 +-
.../org/apache/spark/sql/DataFrameWriter.scala | 25 +-
.../datasources/v2/WriteToDataSourceV2.scala| 8 +-
.../execution/streaming/StreamExecution.scala | 15 +-
.../ContinuousDataSourceRDDIter.scala | 3 +-
.../continuous/ContinuousExecution.scala| 67 ++-
.../streaming/continuous/EpochCoordinator.scala | 21 +-
.../spark/sql/streaming/DataStreamWriter.scala | 26 +-
.../apache/spark/sql/streaming/StreamTest.scala | 36 +-
21 files changed, 383 insertions(+), 1531 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/60bcb468/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
--
diff --git
a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
deleted file mode 100644
index 9283795..000
---
a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.kafka010
-
-import java.{util => ju}
-
-import org.apache.kafka.clients.consumer.ConsumerRecord
-import org.apache.kafka.common.TopicPartition
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.sql.catalyst.expressions.codegen.{BufferHolder,
UnsafeRowWriter}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import
org.apache.spark.sql.kafka010.KafkaSource.{INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_FALSE,
INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_TRUE}
-import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.streaming.reader.{ContinuousDataReader,
ContinuousReader, Offset, PartitionOffset}
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.unsafe.types.UTF8String
-
-/**
- * A [[ContinuousReader]] for data from kafka.
- *
- * @param offsetReader a reader used to get kafka offsets. Note that the
actual data will be
- * read by per-task consumers generated later.
- * @param kafkaParams String params for per-task Kafka consumers.
- * @param sourceOptions The
[[org.apache.spark.sql.sources.v2.DataSourceV2Options]] params which
- * are not Kafka consumer params.
- * @param metadataPath Path to a directory this reader can use for writing
metadata.
- * @param initialOffsets The Kafka offsets to start r
[2/2] spark git commit: Revert "[SPARK-22908] Add kafka source and sink for continuous processing."
Revert "[SPARK-22908] Add kafka source and sink for continuous processing."
This reverts commit 6f7aaed805070d29dcba32e04ca7a1f581fa54b9.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/55dbfbca
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/55dbfbca
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/55dbfbca
Branch: refs/heads/master
Commit: 55dbfbca37ce4c05f83180777ba3d4fe2d96a02e
Parents: 5427739
Author: Sameer Agarwal
Authored: Fri Jan 12 15:00:00 2018 -0800
Committer: Sameer Agarwal
Committed: Fri Jan 12 15:00:00 2018 -0800
--
.../sql/kafka010/KafkaContinuousReader.scala| 232 -
.../sql/kafka010/KafkaContinuousWriter.scala| 119 -
.../spark/sql/kafka010/KafkaOffsetReader.scala | 21 +-
.../apache/spark/sql/kafka010/KafkaSource.scala | 17 +-
.../spark/sql/kafka010/KafkaSourceOffset.scala | 7 +-
.../sql/kafka010/KafkaSourceProvider.scala | 105 +---
.../spark/sql/kafka010/KafkaWriteTask.scala | 71 ++-
.../apache/spark/sql/kafka010/KafkaWriter.scala | 5 +-
.../sql/kafka010/KafkaContinuousSinkSuite.scala | 474 ---
.../kafka010/KafkaContinuousSourceSuite.scala | 96
.../sql/kafka010/KafkaContinuousTest.scala | 64 ---
.../spark/sql/kafka010/KafkaSourceSuite.scala | 470 +-
.../org/apache/spark/sql/DataFrameReader.scala | 32 +-
.../org/apache/spark/sql/DataFrameWriter.scala | 25 +-
.../datasources/v2/WriteToDataSourceV2.scala| 8 +-
.../execution/streaming/StreamExecution.scala | 15 +-
.../ContinuousDataSourceRDDIter.scala | 3 +-
.../continuous/ContinuousExecution.scala| 67 ++-
.../streaming/continuous/EpochCoordinator.scala | 21 +-
.../spark/sql/streaming/DataStreamWriter.scala | 26 +-
.../apache/spark/sql/streaming/StreamTest.scala | 36 +-
21 files changed, 383 insertions(+), 1531 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/55dbfbca/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
--
diff --git
a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
b/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
deleted file mode 100644
index 9283795..000
---
a/external/kafka-0-10-sql/src/main/scala/org/apache/spark/sql/kafka010/KafkaContinuousReader.scala
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.kafka010
-
-import java.{util => ju}
-
-import org.apache.kafka.clients.consumer.ConsumerRecord
-import org.apache.kafka.common.TopicPartition
-
-import org.apache.spark.internal.Logging
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.catalyst.expressions.UnsafeRow
-import org.apache.spark.sql.catalyst.expressions.codegen.{BufferHolder,
UnsafeRowWriter}
-import org.apache.spark.sql.catalyst.util.DateTimeUtils
-import
org.apache.spark.sql.kafka010.KafkaSource.{INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_FALSE,
INSTRUCTION_FOR_FAIL_ON_DATA_LOSS_TRUE}
-import org.apache.spark.sql.sources.v2.reader._
-import org.apache.spark.sql.sources.v2.streaming.reader.{ContinuousDataReader,
ContinuousReader, Offset, PartitionOffset}
-import org.apache.spark.sql.types.StructType
-import org.apache.spark.unsafe.types.UTF8String
-
-/**
- * A [[ContinuousReader]] for data from kafka.
- *
- * @param offsetReader a reader used to get kafka offsets. Note that the
actual data will be
- * read by per-task consumers generated later.
- * @param kafkaParams String params for per-task Kafka consumers.
- * @param sourceOptions The
[[org.apache.spark.sql.sources.v2.DataSourceV2Options]] params which
- * are not Kafka consumer params.
- * @param metadataPath Path to a directory this reader can use for writing
metadata.
- * @param initialOffsets The Kafka offsets to start readi
