Repository: spark Updated Branches: refs/heads/master 8aa5aea7f -> e9b16e67c
[SPARK-7314] [SPARK-3524] [PYSPARK] upgrade Pyrolite to 4.4 This PR upgrades Pyrolite to 4.4, which contains the bug fix for SPARK-3524 and some other performance improvements (e.g., SPARK-6288). The artifact is still under `org.spark-project` on Maven Central since there is no official release published there. Author: Xiangrui Meng <m...@databricks.com> Closes #5850 from mengxr/SPARK-7314 and squashes the following commits: 2ed4a95 [Xiangrui Meng] Merge remote-tracking branch 'apache/master' into SPARK-7314 da3c2dd [Xiangrui Meng] remove my repo fe7e29b [Xiangrui Meng] switch to maven central 6ddac0e [Xiangrui Meng] reverse the machine code for float/double d2d5b5b [Xiangrui Meng] change back to 4.4 7824a9c [Xiangrui Meng] use Pyrolite 3.1 cc3903a [Xiangrui Meng] upgrade Pyrolite to 4.4-0 for testing Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e9b16e67 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e9b16e67 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e9b16e67 Branch: refs/heads/master Commit: e9b16e67c636a8a91ab9fb0f4ef98146abbde1e9 Parents: 8aa5aea Author: Xiangrui Meng <m...@databricks.com> Authored: Mon May 4 23:52:42 2015 -0700 Committer: Xiangrui Meng <m...@databricks.com> Committed: Mon May 4 23:52:42 2015 -0700 ---------------------------------------------------------------------- core/pom.xml | 2 +- .../main/scala/org/apache/spark/api/python/SerDeUtil.scala | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/e9b16e67/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 2dfb00d..c577944 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -362,7 +362,7 @@ <dependency> <groupId>org.spark-project</groupId> <artifactId>pyrolite</artifactId> - <version>2.0.1</version> + <version>4.4</version> </dependency> <dependency> <groupId>net.sf.py4j</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/e9b16e67/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala index 257491e..1f1debc 100644 --- a/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala +++ b/core/src/main/scala/org/apache/spark/api/python/SerDeUtil.scala @@ -56,16 +56,13 @@ private[spark] object SerDeUtil extends Logging { // {'\0', 0, 0, 0} /* Sentinel */ // }; // TODO: support Py_UNICODE with 2 bytes - // FIXME: unpickle array of float is wrong in Pyrolite, so we reverse the - // machine code for float/double here to workaround it. - // we should fix this after Pyrolite fix them val machineCodes: Map[Char, Int] = if (ByteOrder.nativeOrder().equals(ByteOrder.BIG_ENDIAN)) { Map('c' -> 1, 'B' -> 0, 'b' -> 1, 'H' -> 3, 'h' -> 5, 'I' -> 7, 'i' -> 9, - 'L' -> 11, 'l' -> 13, 'f' -> 14, 'd' -> 16, 'u' -> 21 + 'L' -> 11, 'l' -> 13, 'f' -> 15, 'd' -> 17, 'u' -> 21 ) } else { Map('c' -> 1, 'B' -> 0, 'b' -> 1, 'H' -> 2, 'h' -> 4, 'I' -> 6, 'i' -> 8, - 'L' -> 10, 'l' -> 12, 'f' -> 15, 'd' -> 17, 'u' -> 20 + 'L' -> 10, 'l' -> 12, 'f' -> 14, 'd' -> 16, 'u' -> 20 ) } override def construct(args: Array[Object]): Object = { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
