This is an automated email from the ASF dual-hosted git repository. jiayu pushed a commit to branch viz-to-spark in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 03826df93ffbd5224e288706bf880442f00c5c7f Author: Jia Yu <[email protected]> AuthorDate: Wed Sep 6 13:32:51 2023 -0700 Add the initial change --- pom.xml | 2 - spark/common/pom.xml | 17 ++ .../apache/sedona/viz/core/BigBufferedImage.java | 0 .../org/apache/sedona/viz/core/ColoringRule.java | 0 .../apache/sedona/viz/core/GlobalParameter.java | 0 .../org/apache/sedona/viz/core/ImageGenerator.java | 0 .../sedona/viz/core/ImageSerializableWrapper.java | 0 .../org/apache/sedona/viz/core/ImageStitcher.java | 0 .../org/apache/sedona/viz/core/PhotoFilter.java | 0 .../sedona/viz/core/RasterOverlayOperator.java | 0 .../viz/core/Serde/ImageWrapperSerializer.java | 0 .../sedona/viz/core/Serde/PixelSerializer.java | 0 .../viz/core/Serde/SedonaVizKryoRegistrator.java | 0 .../sedona/viz/core/VectorOverlayOperator.java | 0 .../sedona/viz/core/VisualizationOperator.java | 0 .../sedona/viz/core/VisualizationPartitioner.java | 0 .../coloringRule/ColoringRuleFactory.java | 0 .../coloringRule/GenericColoringRule.java | 0 .../viz/extension/coloringRule/LinearFunction.java | 0 .../extension/coloringRule/PiecewiseFunction.java | 0 .../imageGenerator/SedonaVizImageGenerator.java | 0 .../sedona/viz/extension/photoFilter/BoxBlur.java | 0 .../sedona/viz/extension/photoFilter/Embose.java | 0 .../viz/extension/photoFilter/GaussianBlur.java | 0 .../sedona/viz/extension/photoFilter/Outline.java | 0 .../sedona/viz/extension/photoFilter/Sharpen.java | 0 .../visualizationEffect/ChoroplethMap.java | 0 .../viz/extension/visualizationEffect/HeatMap.java | 0 .../extension/visualizationEffect/ScatterPlot.java | 0 .../org/apache/sedona/viz/showcase/Example.java | 0 .../org/apache/sedona/viz/showcase/nycheatmap.png | Bin .../java/org/apache/sedona/viz/showcase/usrail.png | Bin .../org/apache/sedona/viz/showcase/ustweet.png | Bin .../apache/sedona/viz/utils/ColorizeOption.java | 0 .../org/apache/sedona/viz/utils/ImageType.java | 0 .../java/org/apache/sedona/viz/utils/Pixel.java | 0 .../sedona/viz/utils/RasterizationUtils.java | 0 .../org/apache/sedona/viz/utils/S3Operator.java | 0 .../apache/sedona/viz/showcase/ScalaExample.scala | 0 .../sedona/viz/sql/SedonaVizExtensions.scala | 0 .../org/apache/sedona/viz/sql/UDF/Catalog.scala | 0 .../apache/sedona/viz/sql/UDF/UdfRegistrator.scala | 0 .../apache/sedona/viz/sql/UDT/UdtRegistrator.scala | 0 .../sql/operator/AggregateWithinPartitons.scala | 0 .../sedona/viz/sql/operator/VizPartitioner.scala | 0 .../org/apache/sedona/viz/sql/utils/Conf.scala | 0 .../sedona/viz/sql/utils/LineageDecoder.scala | 0 .../viz/sql/utils/SedonaVizRegistrator.scala | 0 .../spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala | 0 .../apache/spark/sql/sedona_viz/UDT/PixelUDT.scala | 0 .../sql/sedona_viz/UDT/UdtRegistratorWrapper.scala | 0 .../sql/sedona_viz/expressions/Colorize.scala | 0 .../sql/sedona_viz/expressions/ImageEncoder.scala | 0 .../sql/sedona_viz/expressions/Partitioner.scala | 0 .../sql/sedona_viz/expressions/Pixelize.scala | 5 +- .../spark/sql/sedona_viz/expressions/Render.scala | 3 +- .../org/apache/sedona/viz/ChoroplethmapTest.java | 0 .../java/org/apache/sedona/viz/HeatmapTest.java | 0 .../org/apache/sedona/viz/NYCTripPointMapper.java | 0 .../java/org/apache/sedona/viz/NYCTripTest.java | 0 .../sedona/viz/ParallelVisualizationTest.java | 0 .../org/apache/sedona/viz/ScatterplotTest.java | 0 .../java/org/apache/sedona/viz/VizTestBase.java | 2 +- .../org/apache/sedona/sql/TestBaseScala.scala | 26 +-- .../org/apache/sedona/viz/rdd/scalaTest.scala | 2 +- .../org/apache/sedona/viz/sql/TestBaseScala.scala | 2 +- .../apache/sedona/viz/sql/optVizOperatorTest.scala | 2 +- .../sedona/viz/sql/standardVizOperatorTest.scala | 0 viz/.gitignore | 10 -- viz/pom.xml | 177 --------------------- viz/src/.gitignore | 1 - viz/src/main/.gitignore | 1 - viz/src/test/.gitignore | 1 - viz/src/test/resources/log4j2.properties | 31 ---- 74 files changed, 40 insertions(+), 242 deletions(-) diff --git a/pom.xml b/pom.xml index 1d83505a8..1bcc4ec37 100644 --- a/pom.xml +++ b/pom.xml @@ -656,7 +656,6 @@ <modules> <module>common</module> <module>spark</module> - <module>viz</module> <module>spark-shaded</module> </modules> </profile> @@ -677,7 +676,6 @@ <modules> <module>common</module> <module>spark</module> - <module>viz</module> <module>spark-shaded</module> <module>flink</module> <module>flink-shaded</module> diff --git a/spark/common/pom.xml b/spark/common/pom.xml index 35899e47d..0a39fbbab 100644 --- a/spark/common/pom.xml +++ b/spark/common/pom.xml @@ -160,6 +160,23 @@ <groupId>org.mockito</groupId> <artifactId>mockito-inline</artifactId> </dependency> + <dependency> + <groupId>org.beryx</groupId> + <artifactId>awt-color-factory</artifactId> + <version>1.0.0</version> + </dependency> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-s3</artifactId> + <version>1.12.362</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> diff --git a/viz/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java b/spark/common/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ColoringRule.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ColoringRule.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/ColoringRule.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/ColoringRule.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java b/spark/common/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java b/spark/common/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java b/spark/common/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java b/spark/common/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java diff --git a/viz/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java b/spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java rename to spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java rename to spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/Example.java b/spark/common/src/main/java/org/apache/sedona/viz/showcase/Example.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/showcase/Example.java rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/Example.java diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png b/spark/common/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/usrail.png b/spark/common/src/main/java/org/apache/sedona/viz/showcase/usrail.png similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/showcase/usrail.png rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/usrail.png diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/ustweet.png b/spark/common/src/main/java/org/apache/sedona/viz/showcase/ustweet.png similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/showcase/ustweet.png rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/ustweet.png diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java rename to spark/common/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/ImageType.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/ImageType.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/utils/ImageType.java rename to spark/common/src/main/java/org/apache/sedona/viz/utils/ImageType.java diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/Pixel.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/Pixel.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/utils/Pixel.java rename to spark/common/src/main/java/org/apache/sedona/viz/utils/Pixel.java diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java rename to spark/common/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/S3Operator.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/S3Operator.java similarity index 100% rename from viz/src/main/java/org/apache/sedona/viz/utils/S3Operator.java rename to spark/common/src/main/java/org/apache/sedona/viz/utils/S3Operator.java diff --git a/viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala b/spark/common/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala similarity index 100% rename from viz/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala similarity index 100% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala similarity index 100% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala similarity index 100% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala similarity index 100% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala similarity index 100% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala similarity index 100% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala similarity index 96% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala index 587b6d4e0..9f7104b35 100644 --- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala @@ -28,10 +28,11 @@ import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback -import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData} +import org.apache.spark.sql.catalyst.util.GenericArrayData import org.apache.spark.sql.sedona_viz.UDT.PixelUDT import org.apache.spark.sql.types.{ArrayType, DataType} -import org.locationtech.jts.geom.{Envelope, Geometry, LineString, MultiLineString, MultiPoint, MultiPolygon, Point, Polygon} +import org.locationtech.jts.geom._ + import scala.jdk.CollectionConverters._ case class ST_Pixelize(inputExpressions: Seq[Expression]) diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala similarity index 98% rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala index d3cfcd6cb..f61215b46 100644 --- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala @@ -18,8 +18,7 @@ */ package org.apache.spark.sql.sedona_viz.expressions -import java.awt.image.BufferedImage // scalastyle:ignore illegal.imports - +import java.awt.image.BufferedImage import org.apache.sedona.viz.core.ImageSerializableWrapper import org.apache.sedona.viz.utils.Pixel import org.apache.spark.internal.Logging diff --git a/viz/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java b/spark/common/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java similarity index 100% rename from viz/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java rename to spark/common/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java diff --git a/viz/src/test/java/org/apache/sedona/viz/HeatmapTest.java b/spark/common/src/test/java/org/apache/sedona/viz/HeatmapTest.java similarity index 100% rename from viz/src/test/java/org/apache/sedona/viz/HeatmapTest.java rename to spark/common/src/test/java/org/apache/sedona/viz/HeatmapTest.java diff --git a/viz/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java b/spark/common/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java similarity index 100% rename from viz/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java rename to spark/common/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java diff --git a/viz/src/test/java/org/apache/sedona/viz/NYCTripTest.java b/spark/common/src/test/java/org/apache/sedona/viz/NYCTripTest.java similarity index 100% rename from viz/src/test/java/org/apache/sedona/viz/NYCTripTest.java rename to spark/common/src/test/java/org/apache/sedona/viz/NYCTripTest.java diff --git a/viz/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java b/spark/common/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java similarity index 100% rename from viz/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java rename to spark/common/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java diff --git a/viz/src/test/java/org/apache/sedona/viz/ScatterplotTest.java b/spark/common/src/test/java/org/apache/sedona/viz/ScatterplotTest.java similarity index 100% rename from viz/src/test/java/org/apache/sedona/viz/ScatterplotTest.java rename to spark/common/src/test/java/org/apache/sedona/viz/ScatterplotTest.java diff --git a/viz/src/test/java/org/apache/sedona/viz/VizTestBase.java b/spark/common/src/test/java/org/apache/sedona/viz/VizTestBase.java similarity index 99% rename from viz/src/test/java/org/apache/sedona/viz/VizTestBase.java rename to spark/common/src/test/java/org/apache/sedona/viz/VizTestBase.java index 923b2a114..29c66cc4f 100644 --- a/viz/src/test/java/org/apache/sedona/viz/VizTestBase.java +++ b/spark/common/src/test/java/org/apache/sedona/viz/VizTestBase.java @@ -147,7 +147,7 @@ public class VizTestBase Logger.getLogger("org").setLevel(Level.WARN); Logger.getLogger("akka").setLevel(Level.WARN); - String resourceFolder = System.getProperty("user.dir") + "/../spark/common/src/test/resources/"; + String resourceFolder = System.getProperty("user.dir") + "/src/test/resources/"; prop = new Properties(); diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala index b54ac4008..662de6d9e 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala @@ -21,9 +21,10 @@ package org.apache.sedona.sql import com.google.common.math.DoubleMath import org.apache.log4j.{Level, Logger} import org.apache.sedona.common.sphere.{Haversine, Spheroid} -import org.apache.sedona.common.Functions.{hausdorffDistance, frechetDistance} +import org.apache.sedona.common.Functions.{frechetDistance, hausdorffDistance} import org.apache.sedona.spark.SedonaContext -import org.apache.spark.sql.DataFrame +import org.apache.spark.SparkContext +import org.apache.spark.sql.{DataFrame, SparkSession} import org.locationtech.jts.geom.{CoordinateSequence, CoordinateSequenceComparator} import org.scalatest.{BeforeAndAfterAll, FunSpec} @@ -35,15 +36,9 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll { Logger.getLogger("org.apache.sedona.core").setLevel(Level.WARN) val warehouseLocation = System.getProperty("user.dir") + "/target/" - val sparkSession = SedonaContext.builder(). - master("local[*]").appName("sedonasqlScalaTest") - .config("spark.sql.warehouse.dir", warehouseLocation) - // We need to be explicit about broadcasting in tests. - .config("sedona.join.autoBroadcastJoinThreshold", "-1") - .config("spark.kryoserializer.buffer.max", "64m") - .getOrCreate() + var sparkSession:SparkSession = null - val sc = sparkSession.sparkContext + var sc: SparkContext = null val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/" val mixedWkbGeometryInputLocation = resourceFolder + "county_small_wkb.tsv" @@ -74,12 +69,21 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll { val smallRasterDataLocation: String = resourceFolder + "raster/test1.tiff" override def beforeAll(): Unit = { + sparkSession = SedonaContext.builder(). + master("local[*]").appName("sedonasqlScalaTest") + .config("spark.sql.warehouse.dir", warehouseLocation) + // We need to be explicit about broadcasting in tests. + .config("sedona.join.autoBroadcastJoinThreshold", "-1") + .config("spark.kryoserializer.buffer.max", "64m") + .getOrCreate() + + sc = sparkSession.sparkContext SedonaContext.create(sparkSession) } override def afterAll(): Unit = { //SedonaSQLRegistrator.dropAll(spark) -// sparkSession.stop + sparkSession.stop } def loadCsv(path: String): DataFrame = { diff --git a/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala similarity index 99% rename from viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala rename to spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala index bc0e65641..068e97ea3 100644 --- a/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala @@ -45,7 +45,7 @@ class scalaTest extends FunSpec with BeforeAndAfterAll{ Logger.getLogger("org").setLevel(Level.WARN) Logger.getLogger("akka").setLevel(Level.WARN) val prop = new Properties() - val resourceFolder = System.getProperty("user.dir") + "/../spark/common/src/test/resources/" + val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/" val demoOutputPath = "target/scala/demo" var ConfFile = new FileInputStream(resourceFolder + "babylon.point.properties") prop.load(ConfFile) diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala similarity index 96% rename from viz/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala rename to spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala index 2967413f3..67bb98346 100644 --- a/viz/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala +++ b/spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala @@ -31,7 +31,7 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll{ Logger.getLogger("org.datasyslab").setLevel(Level.WARN) var spark:SparkSession = _ - val resourceFolder = System.getProperty("user.dir") + "/../spark/common/src/test/resources/" + val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/" val polygonInputLocationWkt = resourceFolder + "county_small.tsv" val polygonInputLocation = resourceFolder + "primaryroads-polygon.csv" diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala similarity index 100% rename from viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala rename to spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala index 9fd048557..8969a1afa 100644 --- a/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala @@ -21,8 +21,8 @@ package org.apache.sedona.viz.sql import org.apache.sedona.viz.sql.operator.{AggregateWithinPartitons, VizPartitioner} import org.apache.sedona.viz.sql.utils.{Conf, LineageDecoder} -import org.locationtech.jts.geom.Envelope import org.apache.spark.sql.functions.lit +import org.locationtech.jts.geom.Envelope class optVizOperatorTest extends TestBaseScala { diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala similarity index 100% rename from viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala rename to spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala diff --git a/viz/.gitignore b/viz/.gitignore deleted file mode 100644 index b2d2d80f2..000000000 --- a/viz/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -/target/ -/.settings/ -/.classpath -/.project -/dependency-reduced-pom.xml -/doc/ -/.idea/ -*.iml -/latest/ -/spark-warehouse/ diff --git a/viz/pom.xml b/viz/pom.xml deleted file mode 100644 index 8d1ae9475..000000000 --- a/viz/pom.xml +++ /dev/null @@ -1,177 +0,0 @@ -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, - ~ software distributed under the License is distributed on an - ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - ~ KIND, either express or implied. See the License for the - ~ specific language governing permissions and limitations - ~ under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.sedona</groupId> - <artifactId>sedona-parent</artifactId> - <version>1.5.0-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - <artifactId>sedona-viz-${spark.compat.version}_${scala.compat.version}</artifactId> - - <name>${project.groupId}:${project.artifactId}</name> - <description>A cluster computing system for processing large-scale spatial data: RDD and SQL for Viz.</description> - <url>http://sedona.apache.org/</url> - <packaging>jar</packaging> - - <properties> - <maven.deploy.skip>false</maven.deploy.skip> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.sedona</groupId> - <artifactId>sedona-common</artifactId> - <version>${project.version}</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.sedona</groupId> - <artifactId>sedona-spark-${spark.compat.version}_${scala.compat.version}</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_${scala.compat.version}</artifactId> - <version>${spark.version}</version> - <scope>provided</scope> - <exclusions> - <!-- Make sure Hadoop's guava version is used for older versions of Spark tests --> - <exclusion> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - </exclusion> - <!-- Exclude log4j 1 for older versions of Spark--> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <!-- Exclude log4j-slf4j2-impl for Spark 3.4 --> - <exclusion> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-slf4j2-impl</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-sql_${scala.compat.version}</artifactId> - <version>${spark.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - </dependency> - <dependency> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-1.2-api</artifactId> - </dependency> - <dependency> - <groupId>org.locationtech.jts</groupId> - <artifactId>jts-core</artifactId> - </dependency> - <dependency> - <groupId>org.geotools</groupId> - <artifactId>gt-main</artifactId> - </dependency> - <dependency> - <groupId>org.geotools</groupId> - <artifactId>gt-referencing</artifactId> - </dependency> - <dependency> - <groupId>org.geotools</groupId> - <artifactId>gt-epsg-hsql</artifactId> - </dependency> - <dependency> - <groupId>org.geotools</groupId> - <artifactId>gt-coverage</artifactId> - </dependency> - <dependency> - <groupId>org.scala-lang</groupId> - <artifactId>scala-library</artifactId> - <version>${scala.version}</version> - </dependency> - <dependency> - <groupId>org.scala-lang.modules</groupId> - <artifactId>scala-collection-compat_${scala.compat.version}</artifactId> - <version>${scala-collection-compat.version}</version> - </dependency> - <dependency> - <groupId>org.beryx</groupId> - <artifactId>awt-color-factory</artifactId> - <version>1.0.0</version> - </dependency> - <dependency> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-s3</artifactId> - <version>1.12.362</version> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>*</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.scalatest</groupId> - <artifactId>scalatest_${scala.compat.version}</artifactId> - <version>${scalatest.version}</version> - <scope>test</scope> - </dependency> - </dependencies> - <build> - <sourceDirectory>src/main/java</sourceDirectory> - <plugins> - <plugin> - <groupId>net.alchim31.maven</groupId> - <artifactId>scala-maven-plugin</artifactId> - <executions> - <execution> - <id>attach-javadocs</id> - <configuration> - <skip>true</skip> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.scalatest</groupId> - <artifactId>scalatest-maven-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.scalastyle</groupId> - <artifactId>scalastyle-maven-plugin</artifactId> - </plugin> - </plugins> - </build> -</project> - diff --git a/viz/src/.gitignore b/viz/src/.gitignore deleted file mode 100644 index 5509140f2..000000000 --- a/viz/src/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.DS_Store diff --git a/viz/src/main/.gitignore b/viz/src/main/.gitignore deleted file mode 100644 index 5509140f2..000000000 --- a/viz/src/main/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.DS_Store diff --git a/viz/src/test/.gitignore b/viz/src/test/.gitignore deleted file mode 100644 index 5509140f2..000000000 --- a/viz/src/test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.DS_Store diff --git a/viz/src/test/resources/log4j2.properties b/viz/src/test/resources/log4j2.properties deleted file mode 100644 index 5f8985946..000000000 --- a/viz/src/test/resources/log4j2.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Set everything to be logged to the file target/unit-tests.log -rootLogger.level = info -rootLogger.appenderRef.file.ref = File - -appender.file.type = File -appender.file.name = File -appender.file.fileName = target/unit-tests.log -appender.file.append = true -appender.file.layout.type = PatternLayout -appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex - -# Ignore messages below warning level from Jetty, because it's a bit verbose -logger.jetty.name = org.sparkproject.jetty -logger.jetty.level = warn
