[
https://issues.apache.org/jira/browse/ORC-203?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16572416#comment-16572416
]
ASF GitHub Bot commented on ORC-203:
------------------------------------
Github user omalley commented on a diff in the pull request:
https://github.com/apache/orc/pull/299#discussion_r208393342
--- Diff: java/core/src/java/org/apache/orc/impl/ColumnStatisticsImpl.java
---
@@ -543,35 +551,73 @@ public void reset() {
super.reset();
minimum = null;
maximum = null;
+ isLowerBoundSet = false;
+ isUpperBoundSet = false;
sum = 0;
}
@Override
public void updateString(Text value) {
if (minimum == null) {
- maximum = minimum = new Text(value);
+ if(value.getLength() > MAX_BYTES_RECORDED) {
+ minimum = truncateLowerBound(value);
+ maximum = truncateUpperBound(value);
+ isLowerBoundSet = true;
+ isUpperBoundSet = true;
+ } else {
+ maximum = minimum = new Text(value);
+ }
} else if (minimum.compareTo(value) > 0) {
- minimum = new Text(value);
+ if(value.getLength() > MAX_BYTES_RECORDED) {
+ minimum = truncateLowerBound(value);
+ isLowerBoundSet = true;
+ }else {
+ minimum = new Text(value);
+ }
} else if (maximum.compareTo(value) < 0) {
- maximum = new Text(value);
+ if(value.getLength() > MAX_BYTES_RECORDED) {
+ maximum = truncateUpperBound(value);
+ isUpperBoundSet = true;
+ } else {
+ maximum = new Text(value);
+ }
}
sum += value.getLength();
}
+
@Override
public void updateString(byte[] bytes, int offset, int length,
int repetitions) {
+ byte[] input = Arrays.copyOfRange(bytes, offset, offset+(length));
if (minimum == null) {
- maximum = minimum = new Text();
- maximum.set(bytes, offset, length);
+ if(length > MAX_BYTES_RECORDED) {
+ minimum = truncateLowerBound(input);
+ maximum = truncateUpperBound(input);
+ isLowerBoundSet = true;
+ isUpperBoundSet = true;
+ } else {
+ maximum = minimum = new Text();
+ maximum.set(bytes, offset, length);
+ }
} else if (WritableComparator.compareBytes(minimum.getBytes(), 0,
minimum.getLength(), bytes, offset, length) > 0) {
- minimum = new Text();
- minimum.set(bytes, offset, length);
+ if(length > MAX_BYTES_RECORDED) {
+ minimum = truncateLowerBound(input);
+ isLowerBoundSet = true;
+ } else {
+ minimum = new Text();
+ minimum.set(bytes, offset, length);
+ }
} else if (WritableComparator.compareBytes(maximum.getBytes(), 0,
maximum.getLength(), bytes, offset, length) < 0) {
- maximum = new Text();
- maximum.set(bytes, offset, length);
+ if(length > MAX_BYTES_RECORDED) {
+ maximum = truncateUpperBound(input);
+ isUpperBoundSet = true;
+ } else {
+ maximum = new Text();
+ maximum.set(bytes, offset, length);
--- End diff --
Clear isUpperBoundSet.
> Modify the StringStatistics to trim minimum and maximum values
> --------------------------------------------------------------
>
> Key: ORC-203
> URL: https://issues.apache.org/jira/browse/ORC-203
> Project: ORC
> Issue Type: Bug
> Reporter: Owen O'Malley
> Assignee: Sandeep More
> Priority: Major
>
> Currently the StringStatistics will record the entire value for minimum or
> maximum. It creates large protobuf objects and serves very little value. I
> think we should trim long strings to 1024 characters and record the fact that
> they were trimmed.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
