This is an automated email from the ASF dual-hosted git repository. malka pushed a commit to branch Sedona-17_Shape&WKBSerDe in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
commit 5f8a441c26256c5f30710037bb1bbf59bc2d6f53 Author: Netanel Malka <[email protected]> AuthorDate: Mon Mar 29 23:47:23 2021 +0300 Added WKBGeometrySerde to handle special geometry types that the Shape serde not handled well --- .../sedona/core/serde/SedonaKryoRegistrator.java | 16 +----- .../core/serde/SedonaKryoRegistratorHelper.java | 8 +-- .../core/serde/SedonaWKBKryoRegistrator.java | 7 +-- .../WKB/WKBGeometrySerde.java} | 59 +++++++++++++++------- .../shape/ShapeGeometrySerde.java} | 8 +-- .../core/serde/spatialindex/SpatialIndexSerde.java | 2 - .../jts/index/quadtree/IndexSerde.java | 6 +-- .../locationtech/jts/index/strtree/IndexSerde.java | 6 +-- ...ySerdeTest.java => ShapeGeometrySerdeTest.java} | 6 +-- 9 files changed, 61 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java index 5843b7d..dfb5962 100644 --- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java +++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistrator.java @@ -20,21 +20,9 @@ package org.apache.sedona.core.serde; import com.esotericsoftware.kryo.Kryo; -import org.apache.log4j.Logger; -import org.apache.sedona.core.geometryObjects.Circle; -import org.apache.sedona.core.geometryObjects.GeometrySerde; +import org.apache.sedona.core.serde.shape.ShapeGeometrySerde; import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde; import org.apache.spark.serializer.KryoRegistrator; -import org.locationtech.jts.geom.Envelope; -import org.locationtech.jts.geom.GeometryCollection; -import org.locationtech.jts.geom.LineString; -import org.locationtech.jts.geom.MultiLineString; -import org.locationtech.jts.geom.MultiPoint; -import org.locationtech.jts.geom.MultiPolygon; -import org.locationtech.jts.geom.Point; -import org.locationtech.jts.geom.Polygon; -import org.locationtech.jts.index.quadtree.Quadtree; -import org.locationtech.jts.index.strtree.STRtree; /** * Register Kryo classes using the Geometry Serde(using the ShapeFile serialization) @@ -45,7 +33,7 @@ public class SedonaKryoRegistrator @Override public void registerClasses(Kryo kryo) { - GeometrySerde serializer = new GeometrySerde(); + ShapeGeometrySerde serializer = new ShapeGeometrySerde(); SpatialIndexSerde indexSerializer = new SpatialIndexSerde(); SedonaKryoRegistratorHelper.registerClasses(kryo, serializer, indexSerializer); diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java index c32a3dc..072d344 100644 --- a/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java +++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaKryoRegistratorHelper.java @@ -23,8 +23,8 @@ import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Serializer; import org.apache.log4j.Logger; import org.apache.sedona.core.geometryObjects.Circle; -import org.apache.sedona.core.geometryObjects.GeometrySerde; -import org.apache.sedona.core.geometryObjects.WKBGeometrySerde; +import org.apache.sedona.core.serde.shape.ShapeGeometrySerde; +import org.apache.sedona.core.serde.WKB.WKBGeometrySerde; import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde; import org.locationtech.jts.geom.*; import org.locationtech.jts.index.quadtree.Quadtree; @@ -43,8 +43,8 @@ public class SedonaKryoRegistratorHelper { Serializer serializer; if (geometrySerdeType instanceof WKBGeometrySerde) { serializer = new WKBGeometrySerde(); - } else if (geometrySerdeType instanceof GeometrySerde) { - serializer = new GeometrySerde(); + } else if (geometrySerdeType instanceof ShapeGeometrySerde) { + serializer = new ShapeGeometrySerde(); } else throw new UnsupportedOperationException(String.format("Geometry Serde: %s is not supported", geometrySerdeType.getClass().getName()) diff --git a/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java b/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java index 939c262..adc5047 100644 --- a/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java +++ b/core/src/main/java/org/apache/sedona/core/serde/SedonaWKBKryoRegistrator.java @@ -20,14 +20,9 @@ package org.apache.sedona.core.serde; import com.esotericsoftware.kryo.Kryo; -import org.apache.log4j.Logger; -import org.apache.sedona.core.geometryObjects.Circle; import org.apache.sedona.core.serde.spatialindex.SpatialIndexSerde; -import org.apache.sedona.core.geometryObjects.WKBGeometrySerde; +import org.apache.sedona.core.serde.WKB.WKBGeometrySerde; import org.apache.spark.serializer.KryoRegistrator; -import org.locationtech.jts.geom.*; -import org.locationtech.jts.index.quadtree.Quadtree; -import org.locationtech.jts.index.strtree.STRtree; /** * Register Kryo classes using the WKB Geometry Serde and SpatialIndexSerde for index objects diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java similarity index 80% copy from core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java copy to core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java index 213e8f2..0543388 100755 --- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java +++ b/core/src/main/java/org/apache/sedona/core/serde/WKB/WKBGeometrySerde.java @@ -17,7 +17,9 @@ * under the License. */ -package org.apache.sedona.core.geometryObjects; +package org.apache.sedona.core.serde.WKB; + +import java.util.Objects; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Registration; @@ -25,7 +27,8 @@ import com.esotericsoftware.kryo.Serializer; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import org.apache.log4j.Logger; -import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde; +import org.apache.sedona.core.geometryObjects.Circle; +import org.apache.sedona.core.serde.shape.ShapeGeometrySerde; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; @@ -36,21 +39,26 @@ import org.locationtech.jts.geom.MultiPoint; import org.locationtech.jts.geom.MultiPolygon; import org.locationtech.jts.geom.Point; import org.locationtech.jts.geom.Polygon; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; /** - * Provides methods to efficiently serialize and deserialize geometry types. + * Provides methods to efficiently serialize and deserialize geometry types using the WKB format * <p> - * Supports Point, LineString, Polygon, MultiPoint, MultiLineString, MultiPolygon, - * GeometryCollection, Circle and Envelope types. + * Supports Point, LineString, Polygon, MultiPoint, MultiLineString, + * MultiPolygon, GeometryCollection, Circle and Envelope types. * <p> - * First byte contains {@link Type#id}. Then go type-specific bytes, followed - * by user-data attached to the geometry. + * First byte contains {@link Type#id}. Then go type-specific bytes, followed by + * user-data attached to the geometry. */ -public class GeometrySerde +public class WKBGeometrySerde extends Serializer { - private static final Logger log = Logger.getLogger(GeometrySerde.class); + // TODO - Add tests for WKB serialization + // TODO - Consolidate GeometrySerde to eliminate code repetition + private static final Logger log = Logger.getLogger(ShapeGeometrySerde.class); private static final GeometryFactory geometryFactory = new GeometryFactory(); @Override @@ -99,13 +107,16 @@ public class GeometrySerde private void writeGeometry(Kryo kryo, Output out, Geometry geometry) { - byte[] data = ShapeSerde.serialize(geometry); + WKBWriter writer = new WKBWriter(2, 2, true); + byte[] data = writer.write(geometry); + + // write geometry length size to read bytes until userData + out.writeInt(data.length, true); out.write(data, 0, data.length); writeUserData(kryo, out, geometry); } - private void writeUserData(Kryo kryo, Output out, Geometry geometry) - { + private void writeUserData(Kryo kryo, Output out, Geometry geometry) { out.writeBoolean(geometry.getUserData() != null); if (geometry.getUserData() != null) { kryo.writeClass(out, geometry.getUserData().getClass()); @@ -118,7 +129,7 @@ public class GeometrySerde { byte typeId = input.readByte(); Type geometryType = Type.fromId(typeId); - switch (geometryType) { + switch (Objects.requireNonNull(geometryType)) { case SHAPE: return readGeometry(kryo, input); case CIRCLE: { @@ -153,8 +164,7 @@ public class GeometrySerde } } - private Object readUserData(Kryo kryo, Input input) - { + private Object readUserData(Kryo kryo, Input input) { Object userData = null; if (input.readBoolean()) { Registration clazz = kryo.readClass(input); @@ -163,10 +173,21 @@ public class GeometrySerde return userData; } - private Geometry readGeometry(Kryo kryo, Input input) - { - Geometry geometry = ShapeSerde.deserialize(input, geometryFactory); - geometry.setUserData(readUserData(kryo, input)); + private Geometry readGeometry(Kryo kryo, Input input) { + WKBReader reader = new WKBReader(); + Geometry geometry; + + int geometryBytesLength = input.readInt(true); + byte[] bytes = input.readBytes(geometryBytesLength); + + try { + geometry = reader.read(bytes); + geometry.setUserData(readUserData(kryo, input)); + } catch (ParseException e) { + log.error("Cannot parse geometry bytes", e); + return null; + } + return geometry; } diff --git a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java b/core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java similarity index 96% rename from core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java rename to core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java index 213e8f2..c5551ed 100755 --- a/core/src/main/java/org/apache/sedona/core/geometryObjects/GeometrySerde.java +++ b/core/src/main/java/org/apache/sedona/core/serde/shape/ShapeGeometrySerde.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.sedona.core.geometryObjects; +package org.apache.sedona.core.serde.shape; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Registration; @@ -26,6 +26,7 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import org.apache.log4j.Logger; import org.apache.sedona.core.formatMapper.shapefileParser.parseUtils.shp.ShapeSerde; +import org.apache.sedona.core.geometryObjects.Circle; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; import org.locationtech.jts.geom.GeometryCollection; @@ -46,11 +47,11 @@ import org.locationtech.jts.geom.Polygon; * First byte contains {@link Type#id}. Then go type-specific bytes, followed * by user-data attached to the geometry. */ -public class GeometrySerde +public class ShapeGeometrySerde extends Serializer { - private static final Logger log = Logger.getLogger(GeometrySerde.class); + private static final Logger log = Logger.getLogger(ShapeGeometrySerde.class); private static final GeometryFactory geometryFactory = new GeometryFactory(); @Override @@ -165,6 +166,7 @@ public class GeometrySerde private Geometry readGeometry(Kryo kryo, Input input) { + //TODO Find the kryo registrator class and create one file of that Geometry geometry = ShapeSerde.deserialize(input, geometryFactory); geometry.setUserData(readUserData(kryo, input)); return geometry; diff --git a/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java b/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java index 084c892..72ca10e 100644 --- a/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java +++ b/core/src/main/java/org/apache/sedona/core/serde/spatialindex/SpatialIndexSerde.java @@ -24,8 +24,6 @@ import com.esotericsoftware.kryo.Serializer; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import org.apache.log4j.Logger; -import org.apache.sedona.core.geometryObjects.GeometrySerde; -import org.apache.sedona.core.geometryObjects.WKBGeometrySerde; import org.locationtech.jts.index.quadtree.IndexSerde; import org.locationtech.jts.index.quadtree.Quadtree; import org.locationtech.jts.index.strtree.STRtree; diff --git a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java b/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java index 5dd6556..6d796af 100644 --- a/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java +++ b/core/src/main/java/org/locationtech/jts/index/quadtree/IndexSerde.java @@ -16,7 +16,7 @@ package org.locationtech.jts.index.quadtree; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.sedona.core.geometryObjects.GeometrySerde; +import org.apache.sedona.core.serde.shape.ShapeGeometrySerde; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -29,9 +29,9 @@ import java.util.List; */ public class IndexSerde { - GeometrySerde geometrySerde; + ShapeGeometrySerde geometrySerde; public IndexSerde() { - geometrySerde = new GeometrySerde(); + geometrySerde = new ShapeGeometrySerde(); } public Object read(Kryo kryo, Input input){ diff --git a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java b/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java index bf3d308..6904b98 100644 --- a/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java +++ b/core/src/main/java/org/locationtech/jts/index/strtree/IndexSerde.java @@ -16,7 +16,7 @@ package org.locationtech.jts.index.strtree; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; -import org.apache.sedona.core.geometryObjects.GeometrySerde; +import org.apache.sedona.core.serde.shape.ShapeGeometrySerde; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; @@ -29,9 +29,9 @@ import java.util.List; */ public class IndexSerde { - GeometrySerde geometrySerde; + ShapeGeometrySerde geometrySerde; public IndexSerde() { - geometrySerde = new GeometrySerde(); + geometrySerde = new ShapeGeometrySerde(); } public Object read(Kryo kryo, Input input){ diff --git a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/ShapeGeometrySerdeTest.java similarity index 95% rename from core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java rename to core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/ShapeGeometrySerdeTest.java index 3bb9af4..ab0e0d4 100644 --- a/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/GeometrySerdeTest.java +++ b/core/src/test/java/org/apache/sedona/core/formatMapper/shapefileParser/shapes/ShapeGeometrySerdeTest.java @@ -23,7 +23,7 @@ import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import org.apache.sedona.core.geometryObjects.Circle; -import org.apache.sedona.core.geometryObjects.GeometrySerde; +import org.apache.sedona.core.serde.shape.ShapeGeometrySerde; import org.apache.sedona.core.utils.GeomUtils; import org.junit.Test; import org.locationtech.jts.geom.Geometry; @@ -35,7 +35,7 @@ import java.io.ByteArrayOutputStream; import static org.junit.Assert.assertTrue; -public class GeometrySerdeTest +public class ShapeGeometrySerdeTest { private final Kryo kryo = new Kryo(); private final WKTReader wktReader = new WKTReader(); @@ -86,7 +86,7 @@ public class GeometrySerdeTest private byte[] serialize(Geometry input) { - kryo.register(input.getClass(), new GeometrySerde()); + kryo.register(input.getClass(), new ShapeGeometrySerde()); ByteArrayOutputStream bos = new ByteArrayOutputStream(); Output output = new Output(bos);
