deniskuzZ commented on code in PR #5952:
URL: https://github.com/apache/hive/pull/5952#discussion_r2198153468
##########
serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryTimestamp.java:
##########
@@ -32,6 +37,13 @@ public class LazyBinaryTimestamp extends
LazyBinaryPrimitive<WritableTimestampObjectInspector, TimestampWritableV2>
{
static final Logger LOG = LoggerFactory.getLogger(LazyBinaryTimestamp.class);
+ // Legacy mode flag
+ private static boolean legacyModeEnabled = false;
Review Comment:
would this work for you?
````
Subject: [PATCH] timestamp
---
Index:
serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryTimestamp.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryTimestamp.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryTimestamp.java
---
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryTimestamp.java
(revision b43e6f6c39b2ae562254af09c13c585ed9252062)
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryTimestamp.java
(date 1752163568289)
@@ -30,16 +30,12 @@
*/
public class LazyBinaryTimestamp extends
LazyBinaryPrimitive<WritableTimestampObjectInspector,
TimestampWritableV2> {
- static final Logger LOG =
LoggerFactory.getLogger(LazyBinaryTimestamp.class);
+ private final boolean legacyConversionEnabled;
- LazyBinaryTimestamp(WritableTimestampObjectInspector oi) {
+ LazyBinaryTimestamp(WritableTimestampObjectInspector oi, boolean
legacyConversionEnabled) {
super(oi);
data = new TimestampWritableV2();
- }
-
- LazyBinaryTimestamp(LazyBinaryTimestamp copy) {
- super(copy);
- data = new TimestampWritableV2(copy.data);
+ this.legacyConversionEnabled = legacyConversionEnabled;
}
/**
@@ -52,6 +48,11 @@
*/
@Override
public void init(ByteArrayRef bytes, int start, int length) {
- data.set(bytes.getData(), start);
+ if (!legacyConversionEnabled) {
+ data.set(bytes.getData(), start);
+ } else {
+ // Convert to java.sql.Timestamp
+ ....
+ }
}
}
Index:
serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
---
a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
(revision b43e6f6c39b2ae562254af09c13c585ed9252062)
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
(date 1752162738087)
@@ -30,8 +30,11 @@
public class LazyBinaryColumnarStruct extends ColumnarStructBase {
- public LazyBinaryColumnarStruct(ObjectInspector oi, List<Integer>
notSkippedColumnIDs) {
+ private final boolean legacyConversionEnabled;
+
+ public LazyBinaryColumnarStruct(ObjectInspector oi, List<Integer>
notSkippedColumnIDs, boolean legacyConversionEnabled) {
super(oi, notSkippedColumnIDs);
+ this.legacyConversionEnabled = legacyConversionEnabled;
}
@Override
@@ -55,6 +58,6 @@
@Override
protected LazyObjectBase createLazyObjectBase(ObjectInspector
objectInspector) {
- return LazyBinaryFactory.createLazyBinaryObject(objectInspector);
+ return LazyBinaryFactory.createLazyBinaryObject(objectInspector,
legacyConversionEnabled);
}
}
Index:
serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
---
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
(revision b43e6f6c39b2ae562254af09c13c585ed9252062)
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
(date 1752162813294)
@@ -58,7 +58,7 @@
* Create a lazy binary primitive class given the type name.
*/
public static LazyBinaryPrimitive<?, ?> createLazyBinaryPrimitiveClass(
- PrimitiveObjectInspector oi) {
+ PrimitiveObjectInspector oi, boolean legacyConversionEnabled) {
PrimitiveCategory p = oi.getPrimitiveCategory();
switch (p) {
case BOOLEAN:
@@ -86,7 +86,7 @@
case DATE:
return new LazyBinaryDate((WritableDateObjectInspector) oi);
case TIMESTAMP:
- return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi);
+ return new LazyBinaryTimestamp((WritableTimestampObjectInspector) oi,
legacyConversionEnabled);
case TIMESTAMPLOCALTZ:
return new
LazyBinaryTimestampLocalTZ((WritableTimestampLocalTZObjectInspector) oi);
case INTERVAL_YEAR_MONTH:
@@ -106,10 +106,14 @@
* Create a hierarchical LazyBinaryObject based on the given typeInfo.
*/
public static LazyBinaryObject createLazyBinaryObject(ObjectInspector oi)
{
+ return createLazyBinaryObject(oi, false);
+ }
+
+ public static LazyBinaryObject createLazyBinaryObject(ObjectInspector oi,
boolean legacyConversionEnabled) {
ObjectInspector.Category c = oi.getCategory();
switch (c) {
case PRIMITIVE:
- return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi);
+ return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi,
legacyConversionEnabled);
case MAP:
return new LazyBinaryMap((LazyBinaryMapObjectInspector) oi);
case LIST:
Index:
serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java
---
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java
(revision b43e6f6c39b2ae562254af09c13c585ed9252062)
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryMap.java
(date 1752162954975)
@@ -272,8 +272,8 @@
if (keyObjects[index] == null) {
// Keys are always primitive
keyObjects[index] = LazyBinaryFactory
- .createLazyBinaryPrimitiveClass((PrimitiveObjectInspector)
((MapObjectInspector) oi)
- .getMapKeyObjectInspector());
+ .createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi
+ .getMapKeyObjectInspector(), false);
}
keyObjects[index].init(bytes, keyStart[index], keyLength[index]);
}
Index:
serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
---
a/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
(revision b43e6f6c39b2ae562254af09c13c585ed9252062)
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
(date 1752163507456)
@@ -22,6 +22,7 @@
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -53,7 +54,7 @@
@Override
public String toString() {
- return getClass().toString()
+ return getClass()
+ "["
+ columnNames
+ ":"
@@ -73,15 +74,16 @@
cachedObjectInspector = LazyBinaryFactory.createColumnarStructInspector(
columnNames, columnTypes);
int size = columnTypes.size();
- List<Integer> notSkipIDs = new ArrayList<Integer>();
- if (!this.configuration.isPresent() ||
ColumnProjectionUtils.isReadAllColumns(this.configuration.get())) {
+ List<Integer> notSkipIDs = new ArrayList<>();
+ if (this.configuration.isEmpty() ||
ColumnProjectionUtils.isReadAllColumns(this.configuration.get())) {
for (int i = 0; i < size; i++) {
notSkipIDs.add(i);
}
} else {
notSkipIDs =
ColumnProjectionUtils.getReadColumnIDs(this.configuration.get());
}
- cachedLazyStruct = new LazyBinaryColumnarStruct(cachedObjectInspector,
notSkipIDs);
+ boolean legacyConversionEnabled = HiveConf.getBoolVar(configuration,
HiveConf.ConfVars.HIVE_RCFILE_TIMESTAMP_LEGACY_CONVERSION);
+ cachedLazyStruct = new LazyBinaryColumnarStruct(cachedObjectInspector,
notSkipIDs, legacyConversionEnabled);
super.initialize(size);
}
@@ -90,7 +92,7 @@
@Override
public Writable serialize(Object obj, ObjectInspector objInspector)
throws SerDeException {
if (objInspector.getCategory() != Category.STRUCT) {
- throw new SerDeException(getClass().toString()
+ throw new SerDeException(getClass()
+ " can only serialize struct types, but we got: "
+ objInspector.getTypeName());
}
@@ -113,7 +115,7 @@
if ((f != null) &&
(foi.getCategory().equals(ObjectInspector.Category.PRIMITIVE))
&& ((PrimitiveObjectInspector) foi).getPrimitiveCategory().equals(
PrimitiveObjectInspector.PrimitiveCategory.STRING)
- && ((StringObjectInspector)
foi).getPrimitiveJavaObject(f).length() == 0) {
+ && ((StringObjectInspector)
foi).getPrimitiveJavaObject(f).isEmpty()) {
serializeStream.write(INVALID_UTF__SINGLE_BYTE, 0, 1);
} else {
LazyBinarySerDe.serialize(serializeStream, f, foi, true,
warnedOnceNullMapKey);
````
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For additional commands, e-mail: gitbox-h...@hive.apache.org
