davidnadeau opened a new issue, #1829: URL: https://github.com/apache/fury/issues/1829
### Search before asking - [X] I had searched in the [issues](https://github.com/apache/fury/issues) and found no similar issues. ### Version 0.7.0 ### Component(s) Java, Other ### Minimal reproduce step FuryUtils.scala ```scala package com.etsy.cachetesting import com.etsy.cachetesting.ListingFeatures import org.apache.fury._ import org.apache.fury.config._ import org.apache.fury.resolver.MetaContext object FuryUtils { val cores = Runtime.getRuntime().availableProcessors() lazy val fury = Fury .builder() .withLanguage(Language.JAVA) .withScalaOptimizationEnabled(true) .requireClassRegistration(true) .withMetaShare(true) .buildThreadSafeFuryPool(cores, cores) fury.setClassLoader(ListingFeatures.getClass.getClassLoader) val context: MetaContext = new MetaContext() def toFury(value: ListingFeatures): Array[Byte] = { fury.execute((f) => { f.getSerializationContext.setMetaContext(context) f.serialize(value) }) } def fromFury(bytes: Array[Byte]): ListingFeatures = { fury.execute((f) => { f.getSerializationContext.setMetaContext(context) f.deserialize(bytes).asInstanceOf[ListingFeatures] }) } fury.register( Class.forName("scala.collection.generic.DefaultSerializationProxy") ) fury.register(Class.forName("scala.collection.immutable.$colon$colon")) fury.register(Class.forName("scala.collection.immutable.Nil$")) fury.register(Class.forName("scala.collection.IterableFactory$ToFactory")) fury.register(Class.forName("scala.collection.immutable.List$")) fury.register(Class.forName("scala.collection.generic.SerializeEnd$")) fury.register(Class.forName("scala.Some")) fury.register(Class.forName("com.etsy.cachetesting.TimestampedUserId")) fury.register(Class.forName("com.etsy.cachetesting.ListingCountFeatures")) fury.register( Class.forName("com.etsy.cachetesting.ListingTimeseriesFeatures") ) fury.register(Class.forName("com.etsy.cachetesting.ListingFeatures")) } ``` ListingFeatures.scala ```scala package com.etsy.cachetesting import java.time.Instant case class TimestampedUserId( timestamp: Instant, userId: Long ) trait FamilyType { def isEmpty: Boolean } case class ListingCountFeatures( feature1: Option[Long] = None, feature2: Option[Long] = None, feature3: Option[Long] = None, feature4: Option[Long] = None ) extends FamilyType { def isEmpty: Boolean = this == ListingCountFeatures() } case class ListingTimeseriesFeatures( feature5: Option[List[TimestampedUserId]] = None, feature6: Option[List[TimestampedUserId]] = None, feature7: Option[List[TimestampedUserId]] = None, feature8: Option[List[TimestampedUserId]] = None ) extends FamilyType { def isEmpty: Boolean = this == ListingTimeseriesFeatures() } case class ListingFeatures( id: Long, counts: Option[ListingCountFeatures] = None, timeseries: Option[ListingTimeseriesFeatures] = None ) ``` Generators.scala (generates data for the test) ```scala package com.etsy.cachetesting import java.time.Instant import org.scalameter.api._ import org.scalameter.picklers.Pickler class Generators()(implicit ev2: Pickler[ListingFeatures] ) { val random = new scala.util.Random(0) val listingFeatures = Gen.single("listingFeatures")( ListingFeatures( id = random.nextLong(), counts = Some( ListingCountFeatures( feature1 = Some(random.nextLong()), feature2 = Some(random.nextLong()), feature3 = Some(random.nextLong()), feature4 = Some(random.nextLong()) ) ), timeseries = Some( ListingTimeseriesFeatures( feature5 = Some( List.fill(random.between(0, 51))( TimestampedUserId(Instant.now(), random.nextLong()) ) ), feature6 = Some( List.fill(random.between(0, 51))( TimestampedUserId(Instant.now(), random.nextLong()) ) ), feature7 = Some( List.fill(random.between(0, 51))( TimestampedUserId(Instant.now(), random.nextLong()) ) ), feature8 = Some( List.fill(random.between(0, 51))( TimestampedUserId(Instant.now(), random.nextLong()) ) ) ) ) ) ) } ``` SerializationBenchmarks.scala ```scala package com.etsy.cachetesting import com.etsy.cachetesting.serialization._ import org.scalameter.api._ object SerializationBenchmarks extends Bench.ForkedTime { import org.scalameter.picklers.noPickler._ measure method "fury serialization round trip" in { exec.benchRuns -> 10000 exec.minWarmupRuns -> 100 exec.maxWarmupRuns -> 100 using( new Generators().listingFeatures ) in { features => FuryUtils.fromFury( FuryUtils.toFury(features) ) } } } ``` build.sbt ```scala ThisBuild / version := "0.0.0" ThisBuild / scalaVersion := "2.13.14" lazy val root = (project in file(".")) .settings( name := "feature-caching-blog", libraryDependencies ++= Seq( "org.apache.fury" % "fury-core" % "0.7.0", "com.storm-enroute" %% "scalameter" % "0.21" ) ) ``` ### What did you expect to see? The benchmark to run. ### What did you see instead? ``` Exception in thread "main" java.lang.OutOfMemoryError: Java heap space at java.base/java.util.Arrays.copyOfRange(Arrays.java:3822) at java.base/java.lang.String.<init>(String.java:529) at java.base/java.lang.String.<init>(String.java:1382) at org.apache.fury.meta.MetaStringDecoder.decode(MetaStringDecoder.java:64) at org.apache.fury.resolver.MetaStringBytes.decode(MetaStringBytes.java:70) at org.apache.fury.resolver.ClassResolver.populateBytesToClassInfo(ClassResolver.java:1709) at org.apache.fury.resolver.ClassResolver.loadBytesToClassInfo(ClassResolver.java:1699) at org.apache.fury.resolver.ClassResolver.readClassInternal(ClassResolver.java:1588) at org.apache.fury.serializer.ReplaceResolveSerializer.readObject(ReplaceResolveSerializer.java:310) at org.apache.fury.serializer.ReplaceResolveSerializer.read(ReplaceResolveSerializer.java:305) at org.apache.fury.Fury.readData(Fury.java:923) at org.apache.fury.serializer.ReplaceResolveSerializer.read(ReplaceResolveSerializer.java:284) at org.apache.fury.serializer.collection.CollectionSerializers$JDKCompatibleCollectionSerializer.read(CollectionSerializers.java:743) at scala.SomeFuryCodec_1_1108411398_717386707.read(SomeFuryCodec_1_1108411398_717386707.java:70) at com.etsy.cachetesting.ListingTimeseriesFeaturesFuryCodec_1_1108411398_36202360.readFields$(ListingTimeseriesFeaturesFuryCodec_1_1108411398_36202360.java:137) at com.etsy.cachetesting.ListingTimeseriesFeaturesFuryCodec_1_1108411398_36202360.read(ListingTimeseriesFeaturesFuryCodec_1_1108411398_36202360.java:171) at scala.SomeFuryCodec_1_1108411398_717386707.read(SomeFuryCodec_1_1108411398_717386707.java:70) at com.etsy.cachetesting.ListingFeaturesFuryCodec_1_1108411398_1589683045.readFields$(ListingFeaturesFuryCodec_1_1108411398_1589683045.java:97) at com.etsy.cachetesting.ListingFeaturesFuryCodec_1_1108411398_1589683045.read(ListingFeaturesFuryCodec_1_1108411398_1589683045.java:118) at org.apache.fury.Fury.readDataInternal(Fury.java:955) at org.apache.fury.Fury.readRef(Fury.java:857) at org.apache.fury.Fury.deserialize(Fury.java:789) at org.apache.fury.Fury.deserialize(Fury.java:711) at com.etsy.cachetesting.serialization.FuryUtils$.$anonfun$fromFury$1(FuryUtils.scala:33) at com.etsy.cachetesting.serialization.FuryUtils$$$Lambda$263/0x0000000800e55b08.apply(Unknown Source) at org.apache.fury.pool.ThreadPoolFury.execute(ThreadPoolFury.java:79) at com.etsy.cachetesting.serialization.FuryUtils$.fromFury(FuryUtils.scala:31) at com.etsy.cachetesting.SerializationBenchmarks$.$anonfun$new$2(SerializationBenchmarks.scala:68) at com.etsy.cachetesting.SerializationBenchmarks$$$Lambda$95/0x0000000800d34618.apply(Unknown Source) at org.scalameter.execution.SeparateJvmsExecutor.$anonfun$runSetup$4(SeparateJvmsExecutor.scala:105) at org.scalameter.execution.SeparateJvmsExecutor.$anonfun$runSetup$4$adapted(SeparateJvmsExecutor.scala:105) at org.scalameter.execution.SeparateJvmsExecutor$$Lambda$102/0x0000000800d84000.apply(Unknown Source) ``` ### Anything Else? This error occurs when forcing class registration. When I turn class registration off, I instead get: ``` Exception in thread "main" java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException: Index 447 out of bounds for length 14 at org.apache.fury.pool.ThreadPoolFury.execute(ThreadPoolFury.java:82) at com.etsy.cachetesting.serialization.FuryUtils$.fromFury(FuryUtils.scala:30) at com.etsy.cachetesting.SerializationBenchmarks$.$anonfun$new$2(SerializationBenchmarks.scala:68) at org.scalameter.execution.SeparateJvmsExecutor.$anonfun$runSetup$4(SeparateJvmsExecutor.scala:105) at org.scalameter.execution.SeparateJvmsExecutor.$anonfun$runSetup$4$adapted(SeparateJvmsExecutor.scala:105) at org.scalameter.Warmer$Default$$anon$2.$anonfun$foreach$5(Warmer.scala:56) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.scalameter.utils.package$.withGCNotification(package.scala:24) at org.scalameter.Warmer$Default$$anon$2.foreach(Warmer.scala:49) at org.scalameter.execution.SeparateJvmsExecutor.$anonfun$runSetup$3(SeparateJvmsExecutor.scala:105) at org.scalameter.execution.SeparateJvmsExecutor.$anonfun$runSetup$3$adapted(SeparateJvmsExecutor.scala:104) at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:619) at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:617) at scala.collection.AbstractIterator.foreach(Iterator.scala:1303) at org.scalameter.execution.SeparateJvmsExecutor.$anonfun$runSetup$1(SeparateJvmsExecutor.scala:104) at org.scalameter.execution.Main$.mainMethod(Main.scala:16) at org.scalameter.execution.Main$.main(Main.scala:10) at org.scalameter.execution.Main.main(Main.scala) Caused by: java.lang.ArrayIndexOutOfBoundsException: Index 447 out of bounds for length 14 at org.apache.fury.collection.ObjectArray.get(ObjectArray.java:62) at org.apache.fury.resolver.ClassResolver.readClassInfoWithMetaShare(ClassResolver.java:1348) at org.apache.fury.resolver.ClassResolver.readClassInfo(ClassResolver.java:1638) at scala.collection.immutable._colon_colonFuryMetaShared8782623466212560208Codec_1_1108411398_237351678.readFields$(_colon_colonFuryMetaShared8782623466212560208Codec_1_1108411398_237351678.java:73) at scala.collection.immutable._colon_colonFuryMetaShared8782623466212560208Codec_1_1108411398_237351678.read(_colon_colonFuryMetaShared8782623466212560208Codec_1_1108411398_237351678.java:122) at scala.SomeFuryMetaShared2848185738884915280Codec_1_1108411398_717386707.read(SomeFuryMetaShared2848185738884915280Codec_1_1108411398_717386707.java:47) at com.etsy.cachetesting.ListingTimeseriesFeaturesFuryMetaShared7510468037580857040Codec_1_1108411398_36202360.readFields$(ListingTimeseriesFeaturesFuryMetaShared7510468037580857040Codec_1_1108411398_36202360.java:51) at com.etsy.cachetesting.ListingTimeseriesFeaturesFuryMetaShared7510468037580857040Codec_1_1108411398_36202360.read(ListingTimeseriesFeaturesFuryMetaShared7510468037580857040Codec_1_1108411398_36202360.java:80) at scala.SomeFuryMetaShared2848185738884915280Codec_1_1108411398_717386707.read(SomeFuryMetaShared2848185738884915280Codec_1_1108411398_717386707.java:47) at com.etsy.cachetesting.ListingFeaturesFuryMetaShared8713304253468170576Codec_1_1108411398_1589683045.readFields$(ListingFeaturesFuryMetaShared8713304253468170576Codec_1_1108411398_1589683045.java:53) at com.etsy.cachetesting.ListingFeaturesFuryMetaShared8713304253468170576Codec_1_1108411398_1589683045.read(ListingFeaturesFuryMetaShared8713304253468170576Codec_1_1108411398_1589683045.java:66) at org.apache.fury.Fury.readDataInternal(Fury.java:955) at org.apache.fury.Fury.readRef(Fury.java:857) at org.apache.fury.Fury.deserialize(Fury.java:789) at org.apache.fury.Fury.deserialize(Fury.java:711) at com.etsy.cachetesting.serialization.FuryUtils$.$anonfun$fromFury$1(FuryUtils.scala:32) at org.apache.fury.pool.ThreadPoolFury.execute(ThreadPoolFury.java:79) ... 17 more ``` ### Are you willing to submit a PR? - [ ] I'm willing to submit a PR! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@fury.apache.org For additional commands, e-mail: commits-h...@fury.apache.org
