[
https://issues.apache.org/jira/browse/PIG-4295?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
liyunzhang_intel updated PIG-4295:
----------------------------------
Attachment: PIG-4295_4.patch
In PIG-4295_4.patch:
fix the problem nothing will be serialize and deserialized in mr/tez mode in
PIG-4295_3.patch.
change code from
{code}
private void writeObject(ObjectOutputStream out) throws IOException {
if(getExecType() instanceof SparkExecType) {
out.defaultWriteObject();
ArrayList<String> udfImportList = new ArrayList<String>();
if (packageImportList.get() == null) {
String udfImportListStr =
properties.getProperty("spark.udf.import.list");
if (udfImportListStr != null) {
udfImportList =
Lists.newArrayList(Splitter.on(",").split(udfImportListStr));
}
} else {
udfImportList = packageImportList.get();
}
out.writeObject(udfImportList);
}
}
{code}
to
{code}
private void writeObject(ObjectOutputStream out) throws IOException {
out.defaultWriteObject();
if(getExecType() instanceof SparkExecType) {
ArrayList<String> udfImportList = new ArrayList<String>();
if (packageImportList.get() == null) {
String udfImportListStr =
properties.getProperty("spark.udf.import.list");
if (udfImportListStr != null) {
udfImportList =
Lists.newArrayList(Splitter.on(",").split(udfImportListStr));
}
} else {
udfImportList = packageImportList.get();
}
out.writeObject(udfImportList);
}
}
{code}
and
{code}
private void readObject(ObjectInputStream in) throws IOException,
ClassNotFoundException {
if(getExecType() instanceof SparkExecType) {
in.defaultReadObject();
ArrayList<String> udfImportList = (ArrayList<String>)
in.readObject();
packageImportList.set(udfImportList);
}
}
{code}
to
{code}
private void readObject(ObjectInputStream in) throws IOException,
ClassNotFoundException {
in.defaultReadObject();
if(getExecType() instanceof SparkExecType) {
ArrayList<String> udfImportList = (ArrayList<String>)
in.readObject();
packageImportList.set(udfImportList);
}
}
{code}
> Enable unit test "TestPigContext" for spark
> -------------------------------------------
>
> Key: PIG-4295
> URL: https://issues.apache.org/jira/browse/PIG-4295
> Project: Pig
> Issue Type: Sub-task
> Components: spark
> Affects Versions: spark-branch
> Reporter: liyunzhang_intel
> Assignee: liyunzhang_intel
> Fix For: spark-branch
>
> Attachments: PIG-4295.patch, PIG-4295_1.patch, PIG-4295_2.patch,
> PIG-4295_3.patch, PIG-4295_4.patch,
> TEST-org.apache.pig.test.TestPigContext.txt
>
>
> error log is attached
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
