Yeah, can you open a JIRA with that reproduction please? You can ping me on it.
On Tue, May 17, 2016 at 4:55 PM, Reynold Xin <r...@databricks.com> wrote: > It seems like the problem here is that we are not using unique names > for mapelements_isNull? > > > > On Tue, May 17, 2016 at 3:29 PM, Koert Kuipers <ko...@tresata.com> wrote: > >> hello all, we are slowly expanding our test coverage for spark >> 2.0.0-SNAPSHOT to more in-house projects. today i ran into this issue... >> >> this runs fine: >> val df = sc.parallelize(List(("1", "2"), ("3", "4"))).toDF("a", "b") >> df >> .map(row => row)(RowEncoder(df.schema)) >> .select("a", "b") >> .show >> >> however this fails: >> val df = sc.parallelize(List(("1", "2"), ("3", "4"))).toDF("a", "b") >> df >> .map(row => row)(RowEncoder(df.schema)) >> .select("b", "a") >> .show >> >> the error is: >> java.lang.Exception: failed to compile: >> org.codehaus.commons.compiler.CompileException: File 'generated.java', Line >> 94, Column 57: Expression "mapelements_isNull" is not an rvalue >> /* 001 */ public Object generate(Object[] references) { >> /* 002 */ return new GeneratedIterator(references); >> /* 003 */ } >> /* 004 */ >> /* 005 */ /** Codegened pipeline for: >> /* 006 */ * Project [b#11,a#10] >> /* 007 */ +- SerializeFromObject [if (input[0, >> org.apache.spark.sql.Row].isNullAt) null else staticinvoke(class org.ap... >> /* 008 */ */ >> /* 009 */ final class GeneratedIterator extends >> org.apache.spark.sql.execution.BufferedRowIterator { >> /* 010 */ private Object[] references; >> /* 011 */ private scala.collection.Iterator inputadapter_input; >> /* 012 */ private UnsafeRow project_result; >> /* 013 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder >> project_holder; >> /* 014 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter >> project_rowWriter; >> /* 015 */ private Object[] deserializetoobject_values; >> /* 016 */ private org.apache.spark.sql.types.StructType >> deserializetoobject_schema; >> /* 017 */ private UnsafeRow deserializetoobject_result; >> /* 018 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder >> deserializetoobject_holder; >> /* 019 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter >> deserializetoobject_rowWriter; >> /* 020 */ private UnsafeRow mapelements_result; >> /* 021 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder >> mapelements_holder; >> /* 022 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter >> mapelements_rowWriter; >> /* 023 */ private UnsafeRow serializefromobject_result; >> /* 024 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder >> serializefromobject_holder; >> /* 025 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter >> serializefromobject_rowWriter; >> /* 026 */ private UnsafeRow project_result1; >> /* 027 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder >> project_holder1; >> /* 028 */ private >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter >> project_rowWriter1; >> /* 029 */ >> /* 030 */ public GeneratedIterator(Object[] references) { >> /* 031 */ this.references = references; >> /* 032 */ } >> /* 033 */ >> /* 034 */ public void init(int index, scala.collection.Iterator >> inputs[]) { >> /* 035 */ partitionIndex = index; >> /* 036 */ inputadapter_input = inputs[0]; >> /* 037 */ project_result = new UnsafeRow(2); >> /* 038 */ this.project_holder = new >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(project_result, >> 64); >> /* 039 */ this.project_rowWriter = new >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(project_holder, >> 2); >> /* 040 */ >> /* 041 */ this.deserializetoobject_schema = >> (org.apache.spark.sql.types.StructType) references[0]; >> /* 042 */ deserializetoobject_result = new UnsafeRow(1); >> /* 043 */ this.deserializetoobject_holder = new >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(deserializetoobject_result, >> 32); >> /* 044 */ this.deserializetoobject_rowWriter = new >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(deserializetoobject_holder, >> 1); >> /* 045 */ mapelements_result = new UnsafeRow(1); >> /* 046 */ this.mapelements_holder = new >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(mapelements_result, >> 32); >> /* 047 */ this.mapelements_rowWriter = new >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(mapelements_holder, >> 1); >> /* 048 */ serializefromobject_result = new UnsafeRow(2); >> /* 049 */ this.serializefromobject_holder = new >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(serializefromobject_result, >> 64); >> /* 050 */ this.serializefromobject_rowWriter = new >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(serializefromobject_holder, >> 2); >> /* 051 */ project_result1 = new UnsafeRow(2); >> /* 052 */ this.project_holder1 = new >> org.apache.spark.sql.catalyst.expressions.codegen.BufferHolder(project_result1, >> 64); >> /* 053 */ this.project_rowWriter1 = new >> org.apache.spark.sql.catalyst.expressions.codegen.UnsafeRowWriter(project_holder1, >> 2); >> /* 054 */ } >> /* 055 */ >> /* 056 */ protected void processNext() throws java.io.IOException { >> /* 057 */ /*** PRODUCE: Project [b#11,a#10] */ >> /* 058 */ >> /* 059 */ /*** PRODUCE: SerializeFromObject [if (input[0, >> org.apache.spark.sql.Row].isNullAt) null else staticinvoke(class >> org.apache.spark.unsafe.types... */ >> /* 060 */ >> /* 061 */ /*** PRODUCE: MapElements <function1>, obj#9: >> org.apache.spark.sql.Row */ >> /* 062 */ >> /* 063 */ /*** PRODUCE: DeserializeToObject createexternalrow(if >> (isnull(a#5)) null else a#5.toString, if (isnull(b#6)) null else >> b#6.toString, StructFi... */ >> /* 064 */ >> /* 065 */ /*** PRODUCE: Project [_1#2 AS a#5,_2#3 AS b#6] */ >> /* 066 */ >> /* 067 */ /*** PRODUCE: INPUT */ >> /* 068 */ >> /* 069 */ while (inputadapter_input.hasNext()) { >> /* 070 */ InternalRow inputadapter_row = (InternalRow) >> inputadapter_input.next(); >> /* 071 */ /*** CONSUME: Project [_1#2 AS a#5,_2#3 AS b#6] */ >> /* 072 */ >> /* 073 */ /*** CONSUME: DeserializeToObject createexternalrow(if >> (isnull(a#5)) null else a#5.toString, if (isnull(b#6)) null else >> b#6.toString, StructFi... */ >> /* 074 */ /* input[0, string] */ >> /* 075 */ /* input[0, string] */ >> /* 076 */ boolean inputadapter_isNull = >> inputadapter_row.isNullAt(0); >> /* 077 */ UTF8String inputadapter_value = inputadapter_isNull ? >> null : (inputadapter_row.getUTF8String(0)); >> /* 078 */ /* input[1, string] */ >> /* 079 */ /* input[1, string] */ >> /* 080 */ boolean inputadapter_isNull1 = >> inputadapter_row.isNullAt(1); >> /* 081 */ UTF8String inputadapter_value1 = inputadapter_isNull1 ? >> null : (inputadapter_row.getUTF8String(1)); >> /* 082 */ >> /* 083 */ /*** CONSUME: MapElements <function1>, obj#9: >> org.apache.spark.sql.Row */ >> /* 084 */ >> /* 085 */ /*** CONSUME: SerializeFromObject [if (input[0, >> org.apache.spark.sql.Row].isNullAt) null else staticinvoke(class >> org.apache.spark.unsafe.types... */ >> /* 086 */ >> /* 087 */ /*** CONSUME: Project [b#11,a#10] */ >> /* 088 */ >> /* 089 */ /*** CONSUME: WholeStageCodegen */ >> /* 090 */ >> /* 091 */ /* input[1, string] */ >> /* 092 */ /* if (input[0, org.apache.spark.sql.Row].isNullAt) null >> else staticinvoke(class org.apache.spark.unsafe.types.UTF8String, >> StringTy... */ >> /* 093 */ /* input[0, org.apache.spark.sql.Row].isNullAt */ >> /* 094 */ boolean serializefromobject_isNull9 = mapelements_isNull >> || false; >> /* 095 */ final boolean serializefromobject_value9 = >> serializefromobject_isNull9 ? false : mapelements_value.isNullAt(1); >> /* 096 */ boolean serializefromobject_isNull8 = false; >> /* 097 */ UTF8String serializefromobject_value8 = null; >> /* 098 */ if (!serializefromobject_isNull9 && >> serializefromobject_value9) { >> /* 099 */ /* null */ >> /* 100 */ final UTF8String serializefromobject_value12 = null; >> /* 101 */ serializefromobject_isNull8 = true; >> /* 102 */ serializefromobject_value8 = >> serializefromobject_value12; >> /* 103 */ } else { >> /* 104 */ /* staticinvoke(class >> org.apache.spark.unsafe.types.UTF8String, StringType, fromString, >> getexternalrowfield(input[0, org.apache.spa... */ >> /* 105 */ /* getexternalrowfield(input[0, >> org.apache.spark.sql.Row], 1, ObjectType(class java.lang.String)) */ >> /* 106 */ if (mapelements_isNull) { >> /* 107 */ throw new RuntimeException("The input external row >> cannot be null."); >> /* 108 */ } >> /* 109 */ >> /* 110 */ if (mapelements_value.isNullAt(1)) { >> /* 111 */ throw new RuntimeException("The 1th field of input >> row cannot be null."); >> /* 112 */ } >> /* 113 */ >> /* 114 */ final java.lang.String serializefromobject_value14 = >> (java.lang.String) mapelements_value.get(1); >> /* 115 */ boolean serializefromobject_isNull13 = false; >> /* 116 */ final UTF8String serializefromobject_value13 = >> serializefromobject_isNull13 ? null : >> org.apache.spark.unsafe.types.UTF8String.fromString(serializefromobject_value14); >> /* 117 */ serializefromobject_isNull13 = >> serializefromobject_value13 == null; >> /* 118 */ serializefromobject_isNull8 = >> serializefromobject_isNull13; >> /* 119 */ serializefromobject_value8 = >> serializefromobject_value13; >> /* 120 */ } >> /* 121 */ /* input[0, string] */ >> /* 122 */ /* if (input[0, org.apache.spark.sql.Row].isNullAt) null >> else staticinvoke(class org.apache.spark.unsafe.types.UTF8String, >> StringTy... */ >> /* 123 */ /* input[0, org.apache.spark.sql.Row].isNullAt */ >> /* 124 */ /* input[0, org.apache.spark.sql.Row] */ >> /* 125 */ /* <function1>.apply */ >> /* 126 */ /* <function1> */ >> /* 127 */ /* expression: <function1> */ >> /* 128 */ Object mapelements_obj = ((Expression) >> references[1]).eval(null); >> /* 129 */ scala.Function1 mapelements_value1 = (scala.Function1) >> mapelements_obj; >> /* 130 */ /* input[0, org.apache.spark.sql.Row] */ >> /* 131 */ /* createexternalrow(if (isnull(input[0, string])) null >> else input[0, string].toString, if (isnull(input[1, string])) null else >> inp... */ >> /* 132 */ deserializetoobject_values = new Object[2]; >> /* 133 */ /* if (isnull(input[0, string])) null else input[0, >> string].toString */ >> /* 134 */ boolean deserializetoobject_isNull1 = false; >> /* 135 */ java.lang.String deserializetoobject_value1 = null; >> /* 136 */ if (!false && inputadapter_isNull) { >> /* 137 */ /* null */ >> /* 138 */ final java.lang.String deserializetoobject_value4 = >> null; >> /* 139 */ deserializetoobject_isNull1 = true; >> /* 140 */ deserializetoobject_value1 = deserializetoobject_value4; >> /* 141 */ } else { >> /* 142 */ /* input[0, string].toString */ >> /* 143 */ boolean deserializetoobject_isNull5 = >> inputadapter_isNull; >> /* 144 */ final java.lang.String deserializetoobject_value5 = >> deserializetoobject_isNull5 ? null : (java.lang.String) >> inputadapter_value.toString(); >> /* 145 */ deserializetoobject_isNull5 = >> deserializetoobject_value5 == null; >> /* 146 */ deserializetoobject_isNull1 = >> deserializetoobject_isNull5; >> /* 147 */ deserializetoobject_value1 = deserializetoobject_value5; >> /* 148 */ } >> /* 149 */ if (deserializetoobject_isNull1) { >> /* 150 */ deserializetoobject_values[0] = null; >> /* 151 */ } else { >> /* 152 */ deserializetoobject_values[0] = >> deserializetoobject_value1; >> /* 153 */ } >> /* 154 */ /* if (isnull(input[1, string])) null else input[1, >> string].toString */ >> /* 155 */ boolean deserializetoobject_isNull7 = false; >> /* 156 */ java.lang.String deserializetoobject_value7 = null; >> /* 157 */ if (!false && inputadapter_isNull1) { >> /* 158 */ /* null */ >> /* 159 */ final java.lang.String deserializetoobject_value10 = >> null; >> /* 160 */ deserializetoobject_isNull7 = true; >> /* 161 */ deserializetoobject_value7 = >> deserializetoobject_value10; >> /* 162 */ } else { >> /* 163 */ /* input[1, string].toString */ >> /* 164 */ boolean deserializetoobject_isNull11 = >> inputadapter_isNull1; >> /* 165 */ final java.lang.String deserializetoobject_value11 = >> deserializetoobject_isNull11 ? null : (java.lang.String) >> inputadapter_value1.toString(); >> /* 166 */ deserializetoobject_isNull11 = >> deserializetoobject_value11 == null; >> /* 167 */ deserializetoobject_isNull7 = >> deserializetoobject_isNull11; >> /* 168 */ deserializetoobject_value7 = >> deserializetoobject_value11; >> /* 169 */ } >> /* 170 */ if (deserializetoobject_isNull7) { >> /* 171 */ deserializetoobject_values[1] = null; >> /* 172 */ } else { >> /* 173 */ deserializetoobject_values[1] = >> deserializetoobject_value7; >> /* 174 */ } >> /* 175 */ >> /* 176 */ final org.apache.spark.sql.Row deserializetoobject_value >> = new >> org.apache.spark.sql.catalyst.expressions.GenericRowWithSchema(deserializetoobject_values, >> this.deserializetoobject_schema); >> /* 177 */ boolean mapelements_isNull = false || false; >> /* 178 */ final org.apache.spark.sql.Row mapelements_value = >> mapelements_isNull ? null : (org.apache.spark.sql.Row) >> mapelements_value1.apply(deserializetoobject_value); >> /* 179 */ mapelements_isNull = mapelements_value == null; >> /* 180 */ >> /* 181 */ boolean serializefromobject_isNull1 = mapelements_isNull >> || false; >> /* 182 */ final boolean serializefromobject_value1 = >> serializefromobject_isNull1 ? false : mapelements_value.isNullAt(0); >> /* 183 */ boolean serializefromobject_isNull = false; >> /* 184 */ UTF8String serializefromobject_value = null; >> /* 185 */ if (!serializefromobject_isNull1 && >> serializefromobject_value1) { >> /* 186 */ /* null */ >> /* 187 */ final UTF8String serializefromobject_value4 = null; >> /* 188 */ serializefromobject_isNull = true; >> /* 189 */ serializefromobject_value = serializefromobject_value4; >> /* 190 */ } else { >> /* 191 */ /* staticinvoke(class >> org.apache.spark.unsafe.types.UTF8String, StringType, fromString, >> getexternalrowfield(input[0, org.apache.spa... */ >> /* 192 */ /* getexternalrowfield(input[0, >> org.apache.spark.sql.Row], 0, ObjectType(class java.lang.String)) */ >> /* 193 */ if (mapelements_isNull) { >> /* 194 */ throw new RuntimeException("The input external row >> cannot be null."); >> /* 195 */ } >> /* 196 */ >> /* 197 */ if (mapelements_value.isNullAt(0)) { >> /* 198 */ throw new RuntimeException("The 0th field of input >> row cannot be null."); >> /* 199 */ } >> /* 200 */ >> /* 201 */ final java.lang.String serializefromobject_value6 = >> (java.lang.String) mapelements_value.get(0); >> /* 202 */ boolean serializefromobject_isNull5 = false; >> /* 203 */ final UTF8String serializefromobject_value5 = >> serializefromobject_isNull5 ? null : >> org.apache.spark.unsafe.types.UTF8String.fromString(serializefromobject_value6); >> /* 204 */ serializefromobject_isNull5 = >> serializefromobject_value5 == null; >> /* 205 */ serializefromobject_isNull = >> serializefromobject_isNull5; >> /* 206 */ serializefromobject_value = serializefromobject_value5; >> /* 207 */ } >> /* 208 */ project_holder1.reset(); >> /* 209 */ >> /* 210 */ project_rowWriter1.zeroOutNullBytes(); >> /* 211 */ >> /* 212 */ if (serializefromobject_isNull8) { >> /* 213 */ project_rowWriter1.setNullAt(0); >> /* 214 */ } else { >> /* 215 */ project_rowWriter1.write(0, serializefromobject_value8); >> /* 216 */ } >> /* 217 */ >> /* 218 */ if (serializefromobject_isNull) { >> /* 219 */ project_rowWriter1.setNullAt(1); >> /* 220 */ } else { >> /* 221 */ project_rowWriter1.write(1, serializefromobject_value); >> /* 222 */ } >> /* 223 */ project_result1.setTotalSize(project_holder1.totalSize()); >> /* 224 */ append(project_result1); >> /* 225 */ if (shouldStop()) return; >> /* 226 */ } >> /* 227 */ } >> /* 228 */ } >> > >