[
https://issues.apache.org/jira/browse/SPARK-18263?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15643611#comment-15643611
]
Sean Owen commented on SPARK-18263:
-----------------------------------
Where is the object defined? first you want to rule out more basic problems,
like it not being serializable. Are you relying on some custom serialization
defined in the registrator? either it has to be defined on the command line in
this case, or at least before you create the session object.
> Configuring spark.kryo.registrator programmatically doesn't take effect
> -----------------------------------------------------------------------
>
> Key: SPARK-18263
> URL: https://issues.apache.org/jira/browse/SPARK-18263
> Project: Spark
> Issue Type: Bug
> Components: Spark Core
> Affects Versions: 2.0.1
> Environment: spark-2.0.1-bin-hadoop2.6
> scala-2.11.8
> Reporter: inred
>
> it run ok with spark-shell --conf
> spark.serializer=org.apache.spark.serializer.KryoSerializer \
> --conf
> spark.kryo.registrator=org.bdgenomics.adam.serialization.ADAMKryoRegistrator \
> but in IDE
> val spark = SparkSession.builder.master("local[*]").appName("Anno
> BDG").getOrCreate()
> spark.conf.set("spark.serializer",
> "org.apache.spark.serializer.KryoSerializer")
> spark.conf.set("spark.kryo.registrator",
> "org.bdgenomics.adam.serialization.ADAMKryoRegistrator")
> it reports the following error:
> java.io.NotSerializableException: org.bdgenomics.formats.avro.AlignmentRecord
> Serialization stack:
> object not serializable (class: org.bdgenomics.formats.avro.AlignmentRecord,
> value: {"readInFragment": 0, "contigName": "chr10", "start": 61758687,
> "oldPosition": null, "end": 61758727, "mapq": 25, "readName":
> "NB501244AR:119:HJY3WBGXY:2:11112:6137:19359", "sequence":
> "AAAATACTGAGACTTATCAGAATTTCAGGCTAAAGCAACC", "qual":
> "AAAAAAEEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEE", "cigar": "40M", "oldCigar": null,
> "basesTrimmedFromStart": 0, "basesTrimmedFromEnd": 0, "readPaired": false,
> "properPair": false, "readMapped": true, "mateMapped": false,
> "failedVendorQualityChecks": false, "duplicateRead": false,
> "readNegativeStrand": false, "mateNegativeStrand": false, "primaryAlignment":
> true, "secondaryAlignment": false, "supplementaryAlignment": false,
> "mismatchingPositions": "40", "origQual": null, "attributes":
> "XT:A:U\tXO:i:0\tXM:i:0\tNM:i:0\tXG:i:0\tX1:i:0\tX0:i:1", "recordGroupName":
> null, "recordGroupSample": null, "mateAlignmentStart": null,
> "mateAlignmentEnd": null, "mateContigName": null, "inferredInsertSize": null})
> at
> org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
> at
> org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
> at
> org.apache.spark.serializer.SerializationStream.writeValue(Serializer.scala:135)
> at
> org.apache.spark.storage.DiskBlockObjectWriter.write(DiskBlockObjectWriter.scala:185)
> at
> org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:150)
> at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:79)
> at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:47)
> at org.apache.spark.scheduler.Task.run(Task.scala:86)
> at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> 2016-11-04 10:30:56 ERROR TaskSetManager:70 - Task 0.0 in stage 2.0 (TID 9)
> had a not serializable result: org.bdgenomics.formats.avro.AlignmentRecord
> Serialization stack:
> object not serializable (class: org.bdgenomics.formats.avro.AlignmentRecord,
> value: {"readInFragment": 0, "contigName": "chr1", "start": 10001,
> "oldPosition": null, "end": 10041, "mapq": 0, "readName":
> "NB501244AR:119:HJY3WBGXY:3:11508:7857:8792", "sequence":
> "AACCCTAACCCTAACCCTAACCCTAACCCTAACCCTAACC", "qual":
> "///E////6E////EEAEEE/EEEEEEEEEEEEAEAAA/A", "cigar": "40M", "oldCigar": null,
> "basesTrimmedFromStart": 0, "basesTrimmedFromEnd": 0, "readPaired": false,
> "properPair": false, "readMapped": true, "mateMapped": false,
> "failedVendorQualityChecks": false, "duplicateRead": false,
> "readNegativeStrand": true, "mateNegativeStrand": false, "primaryAlignment":
> true, "secondaryAlignment": false, "supplementaryAlignment": false,
> "mismatchingPositions": "40", "origQual": null, "attributes":
> "XT:A:R\tXO:i:0\tXM:i:0\tNM:i:0\tXG:i:0\tX0:i:594", "recordGroupName": null,
> "recordGroupSample": null, "mateAlignmentStart": null, "mateAlignmentEnd":
> null, "mateContigName": null, "inferredInsertSize": null}); not retrying
> 2016-11-04 10:30:56 ERROR TaskSetManager:70 - Task 4.0 in stage 2.0 (TID 13)
> had a not serializable result: org.bdgenomics.formats.avro.AlignmentRecord
> Serialization stack:
> object not serializable (class: org.bdgenomics.formats.avro.AlignmentRecord,
> value: {"readInFragment": 0, "contigName": "chr10", "start": 61758687,
> "oldPosition": null, "end": 61758727, "mapq": 25, "readName":
> "NB501244AR:119:HJY3WBGXY:2:11112:6137:19359", "sequence":
> "AAAATACTGAGACTTATCAGAATTTCAGGCTAAAGCAACC", "qual":
> "AAAAAAEEEEEAEEEEEEEEEEEEEEEEEEEEEEEEEEEE", "cigar": "40M", "oldCigar": null,
> "basesTrimmedFromStart": 0, "basesTrimmedFromEnd": 0, "readPaired": false,
> "properPair": false, "readMapped": true, "mateMapped": false,
> "failedVendorQualityChecks": false, "duplicateRead": false,
> "readNegativeStrand": false, "mateNegativeStrand": false, "primaryAlignment":
> true, "secondaryAlignment": false, "supplementaryAlignment": false,
> "mismatchingPositions": "40", "origQual": null, "attributes":
> "XT:A:U\tXO:i:0\tXM:i:0\tNM:i:0\tXG:i:0\tX1:i:0\tX0:i:1", "recordGroupName":
> null, "recordGroupSample": null, "mateAlignmentStart": null,
> "mateAlignmentEnd": null, "mateContigName": null, "inferredInsertSize":
> null}); not retrying
> 2016-11-04 10:30:56 ERROR TaskSetManager:70 - Task 3.0 in stage 2.0 (TID 12)
> had a not serializable result: org.bdgenomics.formats.avro.AlignmentRecord
> Serialization stack:
> object not serializable (class: org.bdgenomics.formats.avro.AlignmentRecord,
> value: {"readInFragment": 0, "contigName": "chr7", "start": 68163823,
> "oldPosition": null, "end": 68163863, "mapq": 0, "readName":
> "NB501244AR:119:HJY3WBGXY:4:21602:16293:18064", "sequence":
> "TGTGAGGGTGTTGCCCAAAAGAGATTAACATTTGAGTCAG", "qual":
> "AAAAAEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE", "cigar": "40M", "oldCigar": null,
> "basesTrimmedFromStart": 0, "basesTrimmedFromEnd": 0, "readPaired": false,
> "properPair": false, "readMapped": true, "mateMapped": false,
> "failedVendorQualityChecks": false, "duplicateRead": false,
> "readNegativeStrand": false, "mateNegativeStrand": false, "primaryAlignment":
> true, "secondaryAlignment": false, "supplementaryAlignment": false,
> "mismatchingPositions": "40", "origQual": null, "attributes":
> "XT:A:R\tXO:i:0\tXM:i:0\tNM:i:0\tXG:i:0\tXA:Z:chr3,-84617448,40M,0;\tX1:i:0\tX0:i:2",
> "recordGroupName": null, "recordGroupSample": null, "mateAlignmentStart":
> null, "mateAlignmentEnd": null, "mateContigName": null, "inferredInsertSize":
> null}); not retrying
> 2016-11-04 10:30:56 ERROR TaskSetManager:70 - Task 2.0 in stage 2.0 (TID 11)
> had a not serializable result: org.bdgenomics.formats.avro.AlignmentRecord
> Serialization stack:
> object not serializable (class: org.bdgenomics.formats.avro.AlignmentRecord,
> value: {"readInFragment": 0, "contigName": "chr4", "start": 181076278,
> "oldPosition": null, "end": 181076318, "mapq": 25, "readName":
> "NB501244AR:119:HJY3WBGXY:2:23302:26459:8305", "sequence":
> "CACTGTGTTTTACTTCTATTTTAAAAAACCTGAAGGCTAT", "qual":
> "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEAAAAA", "cigar": "40M", "oldCigar": null,
> "basesTrimmedFromStart": 0, "basesTrimmedFromEnd": 0, "readPaired": false,
> "properPair": false, "readMapped": true, "mateMapped": false,
> "failedVendorQualityChecks": false, "duplicateRead": false,
> "readNegativeStrand": true, "mateNegativeStrand": false, "primaryAlignment":
> true, "secondaryAlignment": false, "supplementaryAlignment": false,
> "mismatchingPositions": "40", "origQual": null, "attributes":
> "XT:A:U\tXO:i:0\tXM:i:0\tNM:i:0\tXG:i:0\tX1:i:0\tX0:i:1", "recordGroupName":
> null, "recordGroupSample": null, "mateAlignmentStart": null,
> "mateAlignmentEnd": null, "mateContigName": null, "inferredInsertSize":
> null}); not retrying
> Process finished with exit code 1
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
