This is an automated email from the ASF dual-hosted git repository. jiayu pushed a commit to branch SEDONA-495 in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 8d195b4b7e0f23d35334b7b131b33bf07681a9c7 Author: Jia Yu <[email protected]> AuthorDate: Wed Feb 14 17:07:43 2024 -0800 Fix --- .../org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala | 5 ++--- spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala index d7851b11d..eaa3d0295 100644 --- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala +++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.sedona_sql.io.raster -import org.apache.hadoop.fs.{FileStatus, Path} +import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.InternalRow @@ -29,7 +29,6 @@ import org.apache.spark.sql.sources.DataSourceRegister import org.apache.spark.sql.types.StructType import java.io.IOException -import java.nio.file.Paths import java.util.UUID private[spark] class RasterFileFormat extends FileFormat with DataSourceRegister { @@ -82,7 +81,7 @@ private class RasterFileWriter(savePath: String, dataSchema: StructType, context: TaskAttemptContext) extends OutputWriter { - private val hfs = new Path(savePath).getFileSystem(context.getConfiguration) + private val hfs = FileSystem.newInstance(new Path(savePath).toUri, context.getConfiguration) private val rasterFieldIndex = if (rasterOptions.rasterField.isEmpty) getRasterFieldIndex else dataSchema.fieldIndex(rasterOptions.rasterField.get) private def getRasterFieldIndex: Int = { diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala index d5203e6a0..9cc755c48 100644 --- a/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala +++ b/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala @@ -149,7 +149,7 @@ class rasterIOTest extends TestBaseScala with BeforeAndAfter with GivenWhenThen } it("should read geotiff using binary source and write geotiff back to hdfs using raster source") { - var rasterDf = sparkSession.read.format("binaryFile").load(rasterdatalocation) + var rasterDf = sparkSession.read.format("binaryFile").load(rasterdatalocation).repartition(3) val rasterCount = rasterDf.count() rasterDf.write.format("raster").mode(SaveMode.Overwrite).save(hdfsURI + "/raster-written") rasterDf = sparkSession.read.format("binaryFile").load(hdfsURI + "/raster-written/*")
