hive git commit: HIVE-13391 : add an option to LLAP to use keytab to authenticate to read data (Sergey Shelukhin, reviewed by Siddharth Seth) ADDENDUM
Repository: hive
Updated Branches:
refs/heads/master afb4605ab -> 44d1bdb54
HIVE-13391 : add an option to LLAP to use keytab to authenticate to read data
(Sergey Shelukhin, reviewed by Siddharth Seth) ADDENDUM
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44d1bdb5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44d1bdb5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44d1bdb5
Branch: refs/heads/master
Commit: 44d1bdb54bbb1f274dea126e9963e061bafdd766
Parents: afb4605
Author: Sergey Shelukhin
Authored: Thu Jun 9 19:08:16 2016 -0700
Committer: Sergey Shelukhin
Committed: Thu Jun 9 19:08:16 2016 -0700
--
.../src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/44d1bdb5/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
--
diff --git
a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index d3fa6ad..571bb25 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1268,7 +1268,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
try {
method = clazz.getDeclaredMethod("getSubject");
method.setAccessible(true);
- ctor = clazz.getConstructor(Subject.class);
+ ctor = clazz.getDeclaredConstructor(Subject.class);
ctor.setAccessible(true);
} catch (Throwable t) {
error = t.getMessage();
hive git commit: HIVE-13391 : add an option to LLAP to use keytab to authenticate to read data (Sergey Shelukhin, reviewed by Siddharth Seth) ADDENDUM
Repository: hive
Updated Branches:
refs/heads/master b895d9d60 -> afb4605ab
HIVE-13391 : add an option to LLAP to use keytab to authenticate to read data
(Sergey Shelukhin, reviewed by Siddharth Seth) ADDENDUM
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/afb4605a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/afb4605a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/afb4605a
Branch: refs/heads/master
Commit: afb4605abf2cffbb676a666b41676518f54d5d94
Parents: b895d9d
Author: Sergey Shelukhin
Authored: Thu Jun 9 15:29:51 2016 -0700
Committer: Sergey Shelukhin
Committed: Thu Jun 9 15:29:51 2016 -0700
--
.../src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/afb4605a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
--
diff --git
a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index 68fac17..d3fa6ad 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1266,7 +1266,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
java.lang.reflect.Constructor ctor;
String error = null;
try {
- method = clazz.getMethod("getSubject");
+ method = clazz.getDeclaredMethod("getSubject");
method.setAccessible(true);
ctor = clazz.getConstructor(Subject.class);
ctor.setAccessible(true);
[2/3] hive git commit: HIVE-13443 : LLAP: signing for the second state of submit (the event) (Sergey Shelukhin, reviewed by Siddharth Seth)
http://git-wip-us.apache.org/repos/asf/hive/blob/b895d9d6/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
--
diff --git
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
index 1df5253..fe2ced5 100644
---
a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
+++
b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/impl/TaskExecutorTestHelpers.java
@@ -39,6 +39,7 @@ import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.hadoop.shim.DefaultHadoopShim;
import org.apache.tez.runtime.api.impl.ExecutionContextImpl;
+import org.apache.tez.runtime.api.impl.TezEvent;
import org.apache.tez.runtime.task.EndReason;
import org.apache.tez.runtime.task.TaskRunner2Result;
import org.slf4j.Logger;
@@ -51,22 +52,21 @@ public class TaskExecutorTestHelpers {
public static MockRequest createMockRequest(int fragmentNum, int
parallelism, long startTime,
boolean canFinish, long
workTime) {
SubmitWorkRequestProto
-requestProto = createSubmitWorkRequestProto(fragmentNum, parallelism,
-startTime);
+request = createSubmitWorkRequestProto(fragmentNum, parallelism,
startTime);
+return createMockRequest(canFinish, workTime, request);
+ }
+
+ private static MockRequest createMockRequest(boolean canFinish,
+ long workTime, SubmitWorkRequestProto request) {
QueryFragmentInfo queryFragmentInfo = createQueryFragmentInfo(
-requestProto.getWorkSpec().getVertex(),
requestProto.getFragmentNumber());
-MockRequest mockRequest = new MockRequest(requestProto, queryFragmentInfo,
canFinish, workTime);
-return mockRequest;
+request.getWorkSpec().getVertex(), request.getFragmentNumber());
+return new MockRequest(request, queryFragmentInfo, canFinish, workTime,
null);
}
public static TaskExecutorService.TaskWrapper createTaskWrapper(
SubmitWorkRequestProto request, boolean canFinish, int workTime) {
-QueryFragmentInfo queryFragmentInfo = createQueryFragmentInfo(
-request.getWorkSpec().getVertex(), request.getFragmentNumber());
-MockRequest mockRequest = new MockRequest(request, queryFragmentInfo,
canFinish, workTime);
-TaskExecutorService.TaskWrapper
-taskWrapper = new TaskExecutorService.TaskWrapper(mockRequest, null);
-return taskWrapper;
+return new TaskExecutorService.TaskWrapper(
+createMockRequest(canFinish, workTime, request), null);
}
public static QueryFragmentInfo createQueryFragmentInfo(
@@ -145,13 +145,13 @@ public class TaskExecutorTestHelpers {
private final Condition finishedCondition = lock.newCondition();
public MockRequest(SubmitWorkRequestProto requestProto, QueryFragmentInfo
fragmentInfo,
- boolean canFinish, long workTime) {
+ boolean canFinish, long workTime, TezEvent
initialEvent) {
super(requestProto, fragmentInfo, new Configuration(),
new ExecutionContextImpl("localhost"), null, new Credentials(), 0,
mock(AMReporter.class), null, mock(
LlapDaemonExecutorMetrics.class),
mock(KilledTaskHandler.class), mock(
FragmentCompletionHandler.class), new DefaultHadoopShim(), null,
- requestProto.getWorkSpec().getVertex(), null);
+ requestProto.getWorkSpec().getVertex(), initialEvent, null);
this.workTime = workTime;
this.canFinish = canFinish;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b895d9d6/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
index 54fc3af..5dd5e80 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
@@ -33,9 +33,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.llap.registry.ServiceInstance;
-import org.apache.hadoop.hive.llap.registry.ServiceInstanceSet;
-import org.apache.hadoop.hive.llap.registry.impl.LlapRegistryService;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -47,7 +44,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.split.SplitLocationProvider
[3/3] hive git commit: HIVE-13443 : LLAP: signing for the second state of submit (the event) (Sergey Shelukhin, reviewed by Siddharth Seth)
HIVE-13443 : LLAP: signing for the second state of submit (the event) (Sergey
Shelukhin, reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b895d9d6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b895d9d6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b895d9d6
Branch: refs/heads/master
Commit: b895d9d607c4d298adf468965dc39b730dc2c1a8
Parents: 6a59cfd
Author: Sergey Shelukhin
Authored: Thu Jun 9 11:49:25 2016 -0700
Committer: Sergey Shelukhin
Committed: Thu Jun 9 11:49:25 2016 -0700
--
.../hive/llap/ext/TestLlapInputSplit.java |9 +-
.../apache/hadoop/hive/llap/LlapInputSplit.java | 35 +-
.../ext/LlapTaskUmbilicalExternalClient.java| 10 +-
.../daemon/rpc/LlapDaemonProtocolProtos.java| 1234 +-
.../hadoop/hive/llap/NotTezEventHelper.java | 71 +
.../src/protobuf/LlapDaemonProtocol.proto | 18 +
.../hadoop/hive/llap/LlapBaseInputFormat.java | 34 +-
.../llap/daemon/impl/ContainerRunnerImpl.java | 57 +-
.../hive/llap/daemon/impl/LlapTaskReporter.java | 31 +-
.../llap/daemon/impl/TaskRunnerCallable.java| 21 +-
.../daemon/impl/TaskExecutorTestHelpers.java| 26 +-
.../hive/ql/exec/tez/HiveSplitGenerator.java|4 -
.../ql/udf/generic/GenericUDTFGetSplits.java| 79 +-
.../org/apache/tez/dag/api/TaskSpecBuilder.java | 13 +-
14 files changed, 1457 insertions(+), 185 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/b895d9d6/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
--
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
index 1de8aa6..7491222 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/llap/ext/TestLlapInputSplit.java
@@ -59,13 +59,8 @@ public class TestLlapInputSplit {
colDescs.add(new FieldDesc("col2", new TypeDesc(TypeDesc.Type.INT)));
Schema schema = new Schema(colDescs);
-LlapInputSplit split1 = new LlapInputSplit(
-splitNum,
-planBytes,
-fragmentBytes,
-locations,
-schema,
-"hive");
+LlapInputSplit split1 = new LlapInputSplit(splitNum, planBytes,
fragmentBytes, null,
+locations, schema, "hive");
ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(byteOutStream);
split1.write(dataOut);
http://git-wip-us.apache.org/repos/asf/hive/blob/b895d9d6/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
--
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
index 02aedfd..ab11926 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/LlapInputSplit.java
@@ -23,24 +23,26 @@ import java.io.IOException;
import org.apache.hadoop.hive.llap.Schema;
import org.apache.hadoop.mapred.InputSplitWithLocationInfo;
import org.apache.hadoop.mapred.SplitLocationInfo;
-import org.apache.thrift.TDeserializer;
-import org.apache.thrift.TSerializer;
public class LlapInputSplit implements InputSplitWithLocationInfo {
- int splitNum;
- byte[] planBytes;
- byte[] fragmentBytes;
- SplitLocationInfo[] locations;
- Schema schema;
- String llapUser;
+ private int splitNum;
+ private byte[] planBytes;
+ private byte[] fragmentBytes;
+ private SplitLocationInfo[] locations;
+ private Schema schema;
+ private String llapUser;
+ private byte[] fragmentBytesSignature;
public LlapInputSplit() {
}
- public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes,
SplitLocationInfo[] locations, Schema schema, String llapUser) {
+ public LlapInputSplit(int splitNum, byte[] planBytes, byte[] fragmentBytes,
+ byte[] fragmentBytesSignature, SplitLocationInfo[] locations, Schema
schema,
+ String llapUser) {
this.planBytes = planBytes;
this.fragmentBytes = fragmentBytes;
+this.fragmentBytesSignature = fragmentBytesSignature;
this.locations = locations;
this.schema = schema;
this.splitNum = splitNum;
@@ -77,7 +79,9 @@ public class LlapInputSplit implements
InputSplitWithLocationInfo {
return fragmentBytes;
}
-
+ public byte[] getFragmentBytesSignature() {
+return fragmentBytesSignature;
+ }
@Override
public void wri
[1/3] hive git commit: HIVE-13675 : LLAP: add HMAC signatures to LLAPIF splits (Sergey Shelukhin, reviewed by Siddharth Seth)
Repository: hive
Updated Branches:
refs/heads/master f67c862e3 -> b895d9d60
HIVE-13675 : LLAP: add HMAC signatures to LLAPIF splits (Sergey Shelukhin,
reviewed by Siddharth Seth)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6a59cfd5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6a59cfd5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6a59cfd5
Branch: refs/heads/master
Commit: 6a59cfd5e1e724697279b285eebed851c49ecdaa
Parents: f67c862
Author: Sergey Shelukhin
Authored: Thu Jun 9 11:47:41 2016 -0700
Committer: Sergey Shelukhin
Committed: Thu Jun 9 11:47:41 2016 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +
.../apache/hadoop/hive/llap/SubmitWorkInfo.java | 54 +--
.../hive/llap/coordinator/LlapCoordinator.java | 159 +++
.../ext/LlapTaskUmbilicalExternalClient.java| 26 +--
.../hadoop/hive/llap/io/api/LlapProxy.java | 20 +--
.../llap/security/LlapTokenLocalClient.java | 59 ---
.../llap/security/LlapTokenLocalClientImpl.java | 61 +++
.../hadoop/hive/llap/security/LlapSigner.java | 4 +-
.../hive/llap/security/LlapSignerImpl.java | 73 +
.../llap/security/LlapTokenLocalClient.java | 30
.../hive/llap/security/SecretManager.java | 6 +
.../llap/security/SigningSecretManager.java | 1 +
.../apache/hadoop/hive/llap/tez/Converters.java | 9 +-
.../hadoop/hive/llap/tez/TestConverters.java| 2 +-
.../hadoop/hive/llap/LlapBaseInputFormat.java | 90 +--
.../llap/daemon/impl/ContainerRunnerImpl.java | 2 +-
.../hive/llap/security/LlapSignerImpl.java | 60 ---
.../hive/llap/security/TestLlapSignerImpl.java | 7 +-
.../llap/tezplugins/LlapTaskCommunicator.java | 2 +-
ql/pom.xml | 1 +
.../hive/ql/exec/tez/TezSessionState.java | 43 ++---
.../ql/udf/generic/GenericUDTFGetSplits.java| 78 +++--
.../apache/hive/service/server/HiveServer2.java | 10 ++
23 files changed, 540 insertions(+), 262 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/6a59cfd5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index fe69ffa..285caa3 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2888,6 +2888,11 @@ public class HiveConf extends Configuration {
LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap",
false,
"Override if grace join should be allowed to run in llap."),
+LLAP_HS2_ENABLE_COORDINATOR("hive.llap.hs2.coordinator.enabled", true,
+"Whether to create the LLAP coordinator; since execution engine and
container vs llap\n" +
+"settings are both coming from job configs, we don't know at start
whether this should\n" +
+"be created. Default true."),
+
SPARK_CLIENT_FUTURE_TIMEOUT("hive.spark.client.future.timeout",
"60s", new TimeValidator(TimeUnit.SECONDS),
"Timeout for requests from Hive client to remote Spark driver."),
http://git-wip-us.apache.org/repos/asf/hive/blob/6a59cfd5/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java
--
diff --git
a/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java
b/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java
index 6704294..95b0ffc 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/llap/SubmitWorkInfo.java
@@ -28,33 +28,32 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.JobTokenSecretManager;
-import org.apache.tez.runtime.api.impl.TaskSpec;
public class SubmitWorkInfo implements Writable {
- private TaskSpec taskSpec;
private ApplicationId fakeAppId;
private long creationTime;
+ private byte[] vertexSpec, vertexSpecSignature;
// This is used to communicate over the LlapUmbilicalProtocol. Not related
to tokens used to
// talk to LLAP daemons itself via the securit work.
private Token token;
+ private int vertexParallelism;
- public SubmitWorkInfo(TaskSpec taskSpec, ApplicationId fakeAppId, long
creationTime) {
-this.taskSpec = taskSpec;
+ public SubmitWorkInfo(ApplicationId fakeAppId, long creationTime,
+ int vertexParallelism, byte[] vertexSpec, byte[] vertexSpecSignature) {
hive git commit: HIVE-13563 : Hive Streaming does not honor orc.compress.size and orc.stripe.size table properties (Wei Zheng, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/branch-1 df3746565 -> 39decb0bf
HIVE-13563 : Hive Streaming does not honor orc.compress.size and
orc.stripe.size table properties (Wei Zheng, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/39decb0b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/39decb0b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/39decb0b
Branch: refs/heads/branch-1
Commit: 39decb0bfce8a2848fd293c5fc932f610896d31d
Parents: df37465
Author: Wei Zheng
Authored: Thu Jun 9 10:12:19 2016 -0700
Committer: Wei Zheng
Committed: Thu Jun 9 11:26:00 2016 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../apache/hadoop/hive/ql/io/orc/OrcFile.java | 7
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 40 ++--
.../hive/ql/io/orc/TestOrcRecordUpdater.java| 4 ++
4 files changed, 42 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/39decb0b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index c63c2ca..e760ed5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1019,6 +1019,8 @@ public class HiveConf extends Configuration {
"to use dictionary or not will be retained thereafter."),
HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", 256 *
1024,
"Define the default ORC buffer size, in bytes."),
+HIVE_ORC_BASE_DELTA_RATIO("hive.exec.orc.base.delta.ratio", 8, "The ratio
of base writer and\n" +
+"delta writer in terms of STRIPE_SIZE and BUFFER_SIZE."),
HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", true,
"Define the default block padding, which pads stripes to the HDFS
block boundaries."),
HIVE_ORC_BLOCK_PADDING_TOLERANCE("hive.exec.orc.block.padding.tolerance",
0.05f,
http://git-wip-us.apache.org/repos/asf/hive/blob/39decb0b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
index c9ba713..3ca0a6e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFile.java
@@ -491,6 +491,13 @@ public final class OrcFile {
return this;
}
+public int getBufferSize() {
+ return bufferSizeValue;
+}
+
+public long getStripeSize() {
+ return stripeSizeValue;
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/39decb0b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index efe5293..693ffd5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
@@ -249,19 +250,36 @@ public class OrcRecordUpdater implements RecordUpdater {
flushLengths = null;
}
OrcFile.WriterOptions writerOptions = null;
-if (options instanceof OrcOptions) {
- writerOptions = ((OrcOptions) options).getOrcOptions();
-}
-if (writerOptions == null) {
- writerOptions = OrcFile.writerOptions(options.getTableProperties(),
- options.getConfiguration());
-}
-writerOptions.fileSystem(fs).callback(indexBuilder);
-if (!options.isWritingBase()) {
+// If writing delta dirs, we need to make a clone of original options, to
avoid polluting it for
+// the base writer
+if (options.isWritingBase()) {
+ if (options instanceof OrcOptions) {
+writerOptions = ((OrcOptions) options).getOrcOptions();
+ }
+ if (writerOptions == null) {
+writerOptions = OrcFile.writerOptions(options.getTableProperties(),
+options.getConfiguration());
+ }
+} else { // delta writer
+ AcidOutputFormat.Opti
hive git commit: HIVE-13563 : Hive Streaming does not honor orc.compress.size and orc.stripe.size table properties (Wei Zheng, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 c6e7c18c5 -> eb52dade5
HIVE-13563 : Hive Streaming does not honor orc.compress.size and
orc.stripe.size table properties (Wei Zheng, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/eb52dade
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/eb52dade
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/eb52dade
Branch: refs/heads/branch-2.1
Commit: eb52dade5d331aa42460143105581001dd314692
Parents: c6e7c18
Author: Wei Zheng
Authored: Thu Jun 9 10:12:19 2016 -0700
Committer: Wei Zheng
Committed: Thu Jun 9 10:19:09 2016 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
orc/src/java/org/apache/orc/OrcConf.java| 2 +
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 42 +++-
.../hive/ql/io/orc/TestOrcRecordUpdater.java| 4 ++
4 files changed, 39 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/eb52dade/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 33e7663..33b0713 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1206,6 +1206,8 @@ public class HiveConf extends Configuration {
"to use dictionary or not will be retained thereafter."),
HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", 256 *
1024,
"Define the default ORC buffer size, in bytes."),
+HIVE_ORC_BASE_DELTA_RATIO("hive.exec.orc.base.delta.ratio", 8, "The ratio
of base writer and\n" +
+"delta writer in terms of STRIPE_SIZE and BUFFER_SIZE."),
HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", true,
"Define the default block padding, which pads stripes to the HDFS
block boundaries."),
HIVE_ORC_BLOCK_PADDING_TOLERANCE("hive.exec.orc.block.padding.tolerance",
0.05f,
http://git-wip-us.apache.org/repos/asf/hive/blob/eb52dade/orc/src/java/org/apache/orc/OrcConf.java
--
diff --git a/orc/src/java/org/apache/orc/OrcConf.java
b/orc/src/java/org/apache/orc/OrcConf.java
index 6fcbb72..357318d 100644
--- a/orc/src/java/org/apache/orc/OrcConf.java
+++ b/orc/src/java/org/apache/orc/OrcConf.java
@@ -40,6 +40,8 @@ public enum OrcConf {
" number of rows n index entry represents.)"),
BUFFER_SIZE("orc.compress.size", "hive.exec.orc.default.buffer.size",
256 * 1024, "Define the default ORC buffer size, in bytes."),
+ BASE_DELTA_RATIO("orc.base.delta.ratio", "hive.exec.orc.base.delta.ratio", 8,
+ "The ratio of base writer and delta writer in terms of STRIPE_SIZE and
BUFFER_SIZE."),
BLOCK_PADDING("orc.block.padding", "hive.exec.orc.default.block.padding",
true,
"Define whether stripes should be padded to the HDFS block boundaries."),
http://git-wip-us.apache.org/repos/asf/hive/blob/eb52dade/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 4bf2403..e577961 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.orc.impl.AcidStats;
import org.apache.orc.impl.OrcAcidUtils;
+import org.apache.orc.OrcConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -204,19 +205,38 @@ public class OrcRecordUpdater implements RecordUpdater {
flushLengths = null;
}
OrcFile.WriterOptions writerOptions = null;
-if (options instanceof OrcOptions) {
- writerOptions = ((OrcOptions) options).getOrcOptions();
-}
-if (writerOptions == null) {
- writerOptions = OrcFile.writerOptions(options.getTableProperties(),
- options.getConfiguration());
-}
-writerOptions.fileSystem(fs).callback(indexBuilder);
-if (!options.isWritingBase()) {
+// If writing delta dirs, we need to make a clone of original options, to
avoid polluting it for
+// the base writer
+if (options.isWritingBase()) {
+ if (options instanceof OrcOptions) {
+writerOptions = ((OrcOptions) options).getOrcOptions();
+ }
+ if (writerOptions == null) {
+writerOptions = OrcFile.writerOptions(options.
hive git commit: HIVE-13563 : Hive Streaming does not honor orc.compress.size and orc.stripe.size table properties (Wei Zheng, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master c9c0c5d1d -> f67c862e3
HIVE-13563 : Hive Streaming does not honor orc.compress.size and
orc.stripe.size table properties (Wei Zheng, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f67c862e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f67c862e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f67c862e
Branch: refs/heads/master
Commit: f67c862e30cc8c3d230853c688c5c70d30d207f3
Parents: c9c0c5d
Author: Wei Zheng
Authored: Thu Jun 9 10:12:19 2016 -0700
Committer: Wei Zheng
Committed: Thu Jun 9 10:12:19 2016 -0700
--
.../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
orc/src/java/org/apache/orc/OrcConf.java| 2 +
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 42 +++-
.../hive/ql/io/orc/TestOrcRecordUpdater.java| 4 ++
.../clientpositive/tez/acid_globallimit.q.out | 6 +--
5 files changed, 42 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
--
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index bb0ca3a..fe69ffa 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1206,6 +1206,8 @@ public class HiveConf extends Configuration {
"to use dictionary or not will be retained thereafter."),
HIVE_ORC_DEFAULT_BUFFER_SIZE("hive.exec.orc.default.buffer.size", 256 *
1024,
"Define the default ORC buffer size, in bytes."),
+HIVE_ORC_BASE_DELTA_RATIO("hive.exec.orc.base.delta.ratio", 8, "The ratio
of base writer and\n" +
+"delta writer in terms of STRIPE_SIZE and BUFFER_SIZE."),
HIVE_ORC_DEFAULT_BLOCK_PADDING("hive.exec.orc.default.block.padding", true,
"Define the default block padding, which pads stripes to the HDFS
block boundaries."),
HIVE_ORC_BLOCK_PADDING_TOLERANCE("hive.exec.orc.block.padding.tolerance",
0.05f,
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/orc/src/java/org/apache/orc/OrcConf.java
--
diff --git a/orc/src/java/org/apache/orc/OrcConf.java
b/orc/src/java/org/apache/orc/OrcConf.java
index 6fcbb72..357318d 100644
--- a/orc/src/java/org/apache/orc/OrcConf.java
+++ b/orc/src/java/org/apache/orc/OrcConf.java
@@ -40,6 +40,8 @@ public enum OrcConf {
" number of rows n index entry represents.)"),
BUFFER_SIZE("orc.compress.size", "hive.exec.orc.default.buffer.size",
256 * 1024, "Define the default ORC buffer size, in bytes."),
+ BASE_DELTA_RATIO("orc.base.delta.ratio", "hive.exec.orc.base.delta.ratio", 8,
+ "The ratio of base writer and delta writer in terms of STRIPE_SIZE and
BUFFER_SIZE."),
BLOCK_PADDING("orc.block.padding", "hive.exec.orc.default.block.padding",
true,
"Define whether stripes should be padded to the HDFS block boundaries."),
http://git-wip-us.apache.org/repos/asf/hive/blob/f67c862e/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
index 4bf2403..e577961 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRecordUpdater.java
@@ -27,6 +27,7 @@ import java.util.List;
import org.apache.orc.impl.AcidStats;
import org.apache.orc.impl.OrcAcidUtils;
+import org.apache.orc.OrcConf;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -204,19 +205,38 @@ public class OrcRecordUpdater implements RecordUpdater {
flushLengths = null;
}
OrcFile.WriterOptions writerOptions = null;
-if (options instanceof OrcOptions) {
- writerOptions = ((OrcOptions) options).getOrcOptions();
-}
-if (writerOptions == null) {
- writerOptions = OrcFile.writerOptions(options.getTableProperties(),
- options.getConfiguration());
-}
-writerOptions.fileSystem(fs).callback(indexBuilder);
-if (!options.isWritingBase()) {
+// If writing delta dirs, we need to make a clone of original options, to
avoid polluting it for
+// the base writer
+if (options.isWritingBase()) {
+ if (options instanceof OrcOptions) {
+writerOptions = ((OrcOptions) options).getOrcOptions();
+ }
+ if (writerOptions == null) {
+
hive git commit: HIVE-13973: Extend support for other primitive types in windowing expressions (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 185d9c572 -> c6e7c18c5
HIVE-13973: Extend support for other primitive types in windowing expressions
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c6e7c18c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c6e7c18c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c6e7c18c
Branch: refs/heads/branch-2.1
Commit: c6e7c18c51e8dcefa0e3c380469792bb92e8bf63
Parents: 185d9c5
Author: Jesus Camacho Rodriguez
Authored: Thu Jun 9 10:27:01 2016 +0100
Committer: Jesus Camacho Rodriguez
Committed: Thu Jun 9 10:27:01 2016 +0100
--
.../hadoop/hive/ql/parse/PTFTranslator.java | 7 +-
.../clientpositive/windowing_range_multiorder.q | 6 +
.../windowing_range_multiorder.q.out| 324 +++
3 files changed, 335 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/c6e7c18c/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
index 018d8d0..f32d02b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
@@ -599,7 +599,7 @@ public class PTFTranslator {
if (!OI.getCategory().equals(Category.PRIMITIVE)) {
throw new SemanticException(
String.format(
- "Value Boundary expression must be of primitve type. Found: %s",
+ "Value Boundary expression must be of primitive type. Found: %s",
OI.getTypeName()));
}
@@ -608,6 +608,7 @@ public class PTFTranslator {
switch (pC)
{
+case BOOLEAN:
case BYTE:
case DOUBLE:
case FLOAT:
@@ -618,10 +619,12 @@ public class PTFTranslator {
case TIMESTAMP:
case DATE:
case STRING:
+case VARCHAR:
+case CHAR:
break;
default:
throw new SemanticException(
- String.format("Primitve type %s not supported in Value Boundary
expression",
+ String.format("Primitive type %s not supported in Value Boundary
expression",
pC));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c6e7c18c/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
--
diff --git a/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
b/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
index d8ca4d6..24ea1e6 100644
--- a/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
+++ b/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
@@ -32,3 +32,9 @@ select s, si, i, min(i) over (partition by s order by si, i
range between unboun
select s, si, i, avg(i) over (partition by s order by si, i desc range between
unbounded preceding and current row) from over10k limit 100;
select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range
between unbounded preceding and current row) from over10k limit 100;
+
+select bo, rank() over (partition by i order by bo nulls first, b nulls last
range between unbounded preceding and unbounded following) from over10k limit
100;
+
+select CAST(s as CHAR(12)), rank() over (partition by i order by CAST(s as
CHAR(12)) nulls last range between unbounded preceding and unbounded following)
from over10k limit 100;
+
+select CAST(s as VARCHAR(12)), rank() over (partition by i order by CAST(s as
VARCHAR(12)) nulls last range between unbounded preceding and unbounded
following) from over10k limit 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/c6e7c18c/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
--
diff --git
a/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
b/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
index 9910883..5f9c3d6 100644
--- a/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
+++ b/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
@@ -908,3 +908,327 @@ POSTHOOK: Input: default@over10k
258true65568 13.57 65568
258true65579 47.52 65579
258true65603 2.6165603
+PREHOOK: query: select bo, rank() over (partition by i order by bo nulls
first, b nulls last range between unbounded preceding and unbounded following)
from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+ A masked pattern was here
+POSTHOOK: query: select bo, rank() over (parti
hive git commit: HIVE-13973: Extend support for other primitive types in windowing expressions (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 7a4fd3377 -> c9c0c5d1d
HIVE-13973: Extend support for other primitive types in windowing expressions
(Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c9c0c5d1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c9c0c5d1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c9c0c5d1
Branch: refs/heads/master
Commit: c9c0c5d1d3cd32498cd0d802f830804b6e5237d0
Parents: 7a4fd33
Author: Jesus Camacho Rodriguez
Authored: Wed Jun 8 19:41:08 2016 +0100
Committer: Jesus Camacho Rodriguez
Committed: Thu Jun 9 10:25:01 2016 +0100
--
.../hadoop/hive/ql/parse/PTFTranslator.java | 7 +-
.../clientpositive/windowing_range_multiorder.q | 6 +
.../windowing_range_multiorder.q.out| 324 +++
3 files changed, 335 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/c9c0c5d1/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
index 018d8d0..f32d02b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
@@ -599,7 +599,7 @@ public class PTFTranslator {
if (!OI.getCategory().equals(Category.PRIMITIVE)) {
throw new SemanticException(
String.format(
- "Value Boundary expression must be of primitve type. Found: %s",
+ "Value Boundary expression must be of primitive type. Found: %s",
OI.getTypeName()));
}
@@ -608,6 +608,7 @@ public class PTFTranslator {
switch (pC)
{
+case BOOLEAN:
case BYTE:
case DOUBLE:
case FLOAT:
@@ -618,10 +619,12 @@ public class PTFTranslator {
case TIMESTAMP:
case DATE:
case STRING:
+case VARCHAR:
+case CHAR:
break;
default:
throw new SemanticException(
- String.format("Primitve type %s not supported in Value Boundary
expression",
+ String.format("Primitive type %s not supported in Value Boundary
expression",
pC));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c9c0c5d1/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
--
diff --git a/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
b/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
index d8ca4d6..24ea1e6 100644
--- a/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
+++ b/ql/src/test/queries/clientpositive/windowing_range_multiorder.q
@@ -32,3 +32,9 @@ select s, si, i, min(i) over (partition by s order by si, i
range between unboun
select s, si, i, avg(i) over (partition by s order by si, i desc range between
unbounded preceding and current row) from over10k limit 100;
select si, bo, i, f, max(i) over (partition by si, bo order by i, f desc range
between unbounded preceding and current row) from over10k limit 100;
+
+select bo, rank() over (partition by i order by bo nulls first, b nulls last
range between unbounded preceding and unbounded following) from over10k limit
100;
+
+select CAST(s as CHAR(12)), rank() over (partition by i order by CAST(s as
CHAR(12)) nulls last range between unbounded preceding and unbounded following)
from over10k limit 100;
+
+select CAST(s as VARCHAR(12)), rank() over (partition by i order by CAST(s as
VARCHAR(12)) nulls last range between unbounded preceding and unbounded
following) from over10k limit 100;
http://git-wip-us.apache.org/repos/asf/hive/blob/c9c0c5d1/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
--
diff --git
a/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
b/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
index 9910883..5f9c3d6 100644
--- a/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
+++ b/ql/src/test/results/clientpositive/windowing_range_multiorder.q.out
@@ -908,3 +908,327 @@ POSTHOOK: Input: default@over10k
258true65568 13.57 65568
258true65579 47.52 65579
258true65603 2.6165603
+PREHOOK: query: select bo, rank() over (partition by i order by bo nulls
first, b nulls last range between unbounded preceding and unbounded following)
from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+ A masked pattern was here
+POSTHOOK: query: select bo, rank() over (partition by
