This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3aa05c84e40 HIVE-28165: HiveSplitGenerator: send splits through
filesystem instead of RPC in case of big payload (#5449) (Laszlo Bodor reviewed
by Denys Kuzmenko)
3aa05c84e40 is described below
commit 3aa05c84e401cf4b67cf17101233621e77bd7325
Author: Bodor Laszlo <[email protected]>
AuthorDate: Wed Sep 18 10:53:20 2024 +0200
HIVE-28165: HiveSplitGenerator: send splits through filesystem instead of
RPC in case of big payload (#5449) (Laszlo Bodor reviewed by Denys Kuzmenko)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 7 +
.../hive/ql/exec/tez/HiveSplitGenerator.java | 173 +++++++++-
.../hive/ql/udf/generic/GenericUDTFGetSplits.java | 8 +-
.../hive/ql/exec/tez/TestHiveSplitGenerator.java | 351 +++++++++++++++++++++
.../clientpositive/tez_split_serialization.q | 14 +
.../llap/tez_split_serialization.q.out | 44 +++
6 files changed, 587 insertions(+), 10 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 280f9a19116..5252ec8724b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4778,6 +4778,13 @@ public class HiveConf extends Configuration {
"Class to use for calculating available slots during split
generation"),
HIVE_TEZ_GENERATE_CONSISTENT_SPLITS("hive.tez.input.generate.consistent.splits",
true,
"Whether to generate consistent split locations when generating splits
in the AM"),
+
HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD("hive.tez.input.fs.serialization.threshold",
268435456,
+ "When the cummulative size of the splits is larger than this (in
bytes), HiveSplitGenerator"
+ + " will start to serialize splits to tez scratchdir instead of
being sent as RPC payloads directly."
+ + " Default is 256MB."
+ + "-1 disables this feature."),
+
HIVE_TEZ_INPUT_FS_SERIALIZATION_THREADS("hive.tez.input.fs.serialization.threads",
8,
+ "Number of threads used for serializing split payloads to
filesystem."),
HIVE_PREWARM_ENABLED("hive.prewarm.enabled", false, "Enables container
prewarm for Tez(Hadoop 2 only)"),
HIVE_PREWARM_NUM_CONTAINERS("hive.prewarm.numcontainers", 10, "Controls
the number of containers to prewarm for Tez (Hadoop 2 only)"),
HIVE_STAGE_ID_REARRANGE("hive.stageid.rearrange", "none", new
StringSet("none", "idonly", "traverse", "execution"), ""),
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
index 46df46bbdf6..d11f135ccb3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
@@ -22,15 +22,30 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Optional;
+import java.util.Queue;
import java.util.Set;
-
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+import java.nio.ByteBuffer;
import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -56,6 +71,8 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.split.SplitLocationProvider;
import org.apache.hadoop.mapreduce.split.TezMapReduceSplitsGrouper;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.tez.common.TezCommonUtils;
import org.apache.tez.common.TezUtils;
import org.apache.tez.dag.api.TaskLocationHint;
import org.apache.tez.dag.api.VertexLocationHint;
@@ -75,6 +92,7 @@ import
org.apache.tez.runtime.api.events.InputInitializerEvent;
import com.google.common.collect.Lists;
import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* This class is used to generate splits inside the AM on the cluster. It
@@ -136,12 +154,13 @@ public class HiveSplitGenerator extends InputInitializer {
this.numSplits = Optional.empty();
}
- private void prepare(InputInitializerContext initializerContext) throws
IOException, SerDeException {
+ @VisibleForTesting
+ void prepare(InputInitializerContext initializerContext) throws IOException,
SerDeException {
userPayloadProto =
MRInputHelpers.parseMRInputPayload(initializerContext.getInputUserPayload());
this.conf =
TezUtils.createConfFromByteString(userPayloadProto.getConfigurationBytes());
-
+
this.jobConf = new JobConf(conf);
// Read all credentials into the credentials instance stored in JobConf.
@@ -154,6 +173,102 @@ public class HiveSplitGenerator extends InputInitializer {
LOG.info("SplitLocationProvider: " + splitLocationProvider);
}
+ /**
+ * SplitSerializer is a helper class for taking care of serializing splits
to the tez scratch dir
+ * when a size criteria defined by
"hive.tez.input.fs.serialization.threshold" is met.
+ * It utilizes an ExecutorService for parallel writes to prevent a single
split write operation
+ * becoming the bottleneck (as write() is called from a loop currently).
+ */
+ class SplitSerializer implements AutoCloseable {
+ private static final String FILE_PATH_FORMAT =
"%s/events/%s/%d_%s_InputDataInformationEvent_%d";
+
+ // fields needed for filepath
+ private String queryId;
+ private String inputName;
+ private int vertexId;
+ private Path appStagingPath;
+ // filesystem and executor
+ private FileSystem fs;
+ private ExecutorService executor;
+
+ private AtomicBoolean anyTaskFailed;
+ private List<Future<?>> asyncTasks;
+
+ @VisibleForTesting
+ SplitSerializer() throws IOException {
+ if (getContext() == null) {
+ // this typically happens in case the split generation is not in the
Tez AM
+ // in that case we're not interested in this feature, so it's fine to
return here and fail later
+ return;
+ }
+ queryId = jobConf.get(HiveConf.ConfVars.HIVE_QUERY_ID.varname);
+ inputName = getContext().getInputName();
+ vertexId = getContext().getVertexId();
+ appStagingPath = TezCommonUtils.getTezSystemStagingPath(jobConf,
getContext().getApplicationId().toString());
+
+ fs = appStagingPath.getFileSystem(jobConf);
+
+ int numThreads = HiveConf.getIntVar(jobConf,
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THREADS);
+ executor = Executors.newFixedThreadPool(numThreads,
+ new
ThreadFactoryBuilder().setNameFormat("HiveSplitGenerator.SplitSerializer Thread
- " + "#%d").build());
+ anyTaskFailed = new AtomicBoolean(false);
+ asyncTasks = new ArrayList<>();
+ }
+
+ @VisibleForTesting
+ InputDataInformationEvent write(int count, MRSplitProto mrSplit) {
+ InputDataInformationEvent diEvent;
+ Path filePath = getSerializedFilePath(count);
+
+ Runnable task = () -> {
+ if (!anyTaskFailed.get()) {
+ try {
+ writeSplit(count, mrSplit, filePath);
+ } catch (IOException e) {
+ anyTaskFailed.set(true);
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ asyncTasks.add(CompletableFuture.runAsync(task, executor));
+ return InputDataInformationEvent.createWithSerializedPath(count,
filePath.toString());
+ }
+
+ @VisibleForTesting
+ void writeSplit(int count, MRSplitProto mrSplit, Path filePath) throws
IOException {
+ long fileWriteStarted = Time.monotonicNow();
+ try (FSDataOutputStream out = fs.create(filePath, false)) {
+ mrSplit.writeTo(out);
+ }
+ LOG.debug("Split #{} event to output path: {} written in {} ms", count,
filePath,
+ fileWriteStarted - Time.monotonicNow());
+ }
+
+ Path getSerializedFilePath(int index) {
+ // e.g. staging_dir/events/queryid/inputtable_InputDataInformationEvent_0
+ return new Path(String.format(FILE_PATH_FORMAT, appStagingPath, queryId,
vertexId, inputName, index));
+ }
+
+ @Override
+ public void close() {
+ try {
+ if (asyncTasks != null) {
+ CompletableFuture.allOf(asyncTasks.toArray(new
CompletableFuture[0])).get();
+ }
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted while generating splits", e);
+ Thread.currentThread().interrupt();
+ } catch (ExecutionException e) {// ExecutionException wraps the original
exception
+ LOG.error("Exception while generating splits", e.getCause());
+ throw new RuntimeException(e.getCause());
+ } finally {
+ if (executor != null) {
+ executor.shutdown();
+ }
+ }
+ }
+ }
+
@SuppressWarnings("unchecked")
@Override
public List<Event> initialize() throws Exception {
@@ -351,7 +466,9 @@ public class HiveSplitGenerator extends InputInitializer {
return splits;
}
- private List<Event> createEventList(boolean sendSerializedEvents,
InputSplitInfoMem inputSplitInfo) {
+ @VisibleForTesting
+ List<Event> createEventList(boolean sendSerializedEvents, InputSplitInfoMem
inputSplitInfo)
+ throws IOException {
List<Event> events =
Lists.newArrayListWithCapacity(inputSplitInfo.getNumTasks() + 1);
@@ -362,13 +479,46 @@ public class HiveSplitGenerator extends InputInitializer {
events.add(configureVertexEvent);
if (sendSerializedEvents) {
- MRSplitsProto splitsProto = inputSplitInfo.getSplitsProto();
int count = 0;
- for (MRSplitProto mrSplit : splitsProto.getSplitsList()) {
- InputDataInformationEvent diEvent =
InputDataInformationEvent.createWithSerializedPayload(
- count++, mrSplit.toByteString().asReadOnlyByteBuffer());
- events.add(diEvent);
+ long inMemoryPayloadSize = 0;
+ long serializedPayloadSize = 0;
+
+ int payloadSerializationThresholdBytes =
+ HiveConf.getIntVar(jobConf,
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD);
+
+ List<MRSplitProto> splits =
inputSplitInfo.getSplitsProto().getSplitsList();
+ LOG.debug("Start creating events for {} splits", splits.size());
+ long startTime = Time.monotonicNow();
+
+ try (SplitSerializer splitSerializer = getSplitSerializer()) {
+ for (MRSplitProto mrSplit : splits) {
+ ByteBuffer payloadBuffer =
mrSplit.toByteString().asReadOnlyByteBuffer();
+ int payloadSize = payloadBuffer.limit();
+ boolean shouldSerializeEventToFile =
+ payloadSerializationThresholdBytes != -1 && inMemoryPayloadSize
>= payloadSerializationThresholdBytes;
+ LOG.debug("Split #{}, byteBuffer size: {} bytes", count,
payloadSize);
+
+ InputDataInformationEvent diEvent = null;
+
+ if (shouldSerializeEventToFile) {
+ LOG.debug("Serialize to path: {}, current in-memory payload: {}
bytes >= threshold: {} bytes",
+ splitSerializer.getSerializedFilePath(count),
inMemoryPayloadSize, payloadSerializationThresholdBytes);
+ serializedPayloadSize += payloadSize;
+ diEvent = splitSerializer.write(count, mrSplit);
+ } else {
+ inMemoryPayloadSize += payloadSize;
+ diEvent =
InputDataInformationEvent.createWithSerializedPayload(count, payloadBuffer);
+ }
+ events.add(diEvent);
+ count += 1;
+ }
}
+
+ // this is useful for making decisions regarding split serialization
threshold
+ long elapsed = Time.monotonicNow() - startTime;
+ LOG.info(
+ "Finished creating events ({} splits) in {} ms, size of payloads: in
memory: {} bytes, serialized to fs: {} bytes",
+ splits.size(), elapsed, inMemoryPayloadSize, serializedPayloadSize);
} else {
int count = 0;
for (org.apache.hadoop.mapred.InputSplit split :
inputSplitInfo.getOldFormatSplits()) {
@@ -380,6 +530,11 @@ public class HiveSplitGenerator extends InputInitializer {
return events;
}
+ @VisibleForTesting
+ SplitSerializer getSplitSerializer() throws IOException {
+ return new SplitSerializer();
+ }
+
@Override
public void onVertexStateUpdated(VertexStateUpdate stateUpdate) {
// pruner registers for vertex state updates after it is ready to handle
them
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index b1c3b767dec..bb029442d66 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -465,7 +465,13 @@ public class GenericUDTFGetSplits extends GenericUDTF {
Preconditions.checkState(HiveConf.getBoolVar(wxConf,
ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS));
- HiveSplitGenerator splitGenerator = new HiveSplitGenerator(wxConf,
mapWork, false, inputArgNumSplits);
+ // we're not interested in split fs serialization optimization in this
case
+ // it was implemented for split generation in TezAM
+ // this can be removed any time when it turns out that's needed here too
+ HiveConf.setIntVar(wxConf,
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, -1);
+
+ HiveSplitGenerator splitGenerator =
+ new HiveSplitGenerator(wxConf, mapWork, false, inputArgNumSplits);
List<Event> eventList = splitGenerator.initialize();
int numGroupedSplitsGenerated = eventList.size() - 1;
InputSplit[] result = new InputSplit[numGroupedSplitsGenerated];
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHiveSplitGenerator.java
b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHiveSplitGenerator.java
new file mode 100644
index 00000000000..8780cb67cd4
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHiveSplitGenerator.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator.SplitSerializer;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat.HiveInputSplit;
+import org.apache.hadoop.hive.ql.io.NullRowsInputFormat;
+import org.apache.hadoop.hive.ql.io.ZeroRowsInputFormat;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.mapreduce.hadoop.InputSplitInfoMem;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.InputInitializerContext;
+import org.apache.tez.runtime.api.events.InputDataInformationEvent;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestHiveSplitGenerator {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TestHiveSplitGenerator.class);
+
+ private static final String QUERY_ID = "hive_" + System.currentTimeMillis();
+ private static final ApplicationId APP_ID = ApplicationId.newInstance(1000,
200);
+ private static final String INPUT_NAME = "MRInput";
+ private static final int VERTEX_ID = 0;
+ private static final String PATH_FORMAT =
"events/%s/%d_%s_InputDataInformationEvent_%d";
+
+ @Test
+ public void testSplitSerializationUsingFileSystem() throws Exception {
+ JobConf conf = new JobConf(new HiveConf());
+ // 0 bytes will force a split serialization to filesystem
+ HiveConf.setIntVar(conf,
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, 0);
+
+ InputSplit[] splits = getSplits(1);
+ String serializedPath = ((InputDataInformationEvent) generateEvents(conf,
splits).get(1)).getSerializedPath();
+ // second split is serialized
+ Assert.assertTrue(
+ "Serialized path should contain a pattern like "
+ +
"'events/$queryid/$vertexid_$inputname_InputDataInformationEvent_$count', but
found: " + serializedPath,
+ serializedPath.contains(expectedSerializedPathFormat(0)));
+ }
+
+ @Test
+ public void testSplitSerializationInMemoryLimitExceeded() throws Exception {
+ JobConf conf = new JobConf(new HiveConf());
+ // split size is around 200 bytes, so first one will be held in memory,
second will be serialized (200 > 10)
+ // this is the most typical usecase of the split fs serialization feature
+ HiveConf.setIntVar(conf,
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, 10);
+
+ InputSplit[] splits = getSplits(2);
+ List<Event> events = generateEvents(conf, splits);
+ String serializedPath1 = ((InputDataInformationEvent)
events.get(1)).getSerializedPath();
+ String serializedPath2 = ((InputDataInformationEvent)
events.get(2)).getSerializedPath();
+
+ // first split is held in memory, no serialized path found
+ Assert.assertNull(serializedPath1);
+
+ // second split is serialized
+ Assert.assertTrue(
+ "Serialized path should contain a pattern like "
+ +
"'events/$queryid/$vertexid_$inputname_InputDataInformationEvent_$count', but
found: " + serializedPath2,
+ serializedPath2.contains(expectedSerializedPathFormat(1)));
+ }
+
+ @Test
+ public void testSplitSerializationNoFileSystemBecauseOfDefaultConfig()
throws Exception {
+ JobConf conf = new JobConf(new HiveConf());
+
+ InputSplit[] splits = getSplits(1);
+ String serializedPath = ((InputDataInformationEvent) generateEvents(conf,
splits).get(1)).getSerializedPath();
+ // the default value 512KB is to large to trigger split serialization to
filesystem
+ Assert.assertNull(serializedPath);
+ }
+
+ @Test
+ public void testSplitSerializationDisabledFilesystem() throws Exception {
+ JobConf conf = new JobConf(new HiveConf());
+ // -1 disables the split serialization to filesystem
+ HiveConf.setIntVar(conf,
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, -1);
+
+ InputSplit[] splits = getSplits(1);
+ String serializedPath = ((InputDataInformationEvent) generateEvents(conf,
splits).get(1)).getSerializedPath();
+ Assert.assertNull(serializedPath);
+ }
+
+ @Test
+ public void testExceptionIsPropagatedFromSplitSerializer() throws Exception {
+ JobConf conf = new JobConf(new HiveConf());
+ HiveConf.setIntVar(conf,
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, 0);
+
+ InputSplit[] splits = getSplits(3);
+ InputInitializerContext context = getInputInitializerContext(conf);
+ HiveSplitGeneratorSerializerException splitGenerator = new
HiveSplitGeneratorSerializerException(context);
+ splitGenerator.prepare(context);
+ try {
+ String serializedPath =
+ ((InputDataInformationEvent) generateEvents(conf, splits,
splitGenerator).get(1)).getSerializedPath();
+ Assert.fail("HiveSplitGeneratorSerializerException should fail");
+ } catch (Exception e) {
+ // outermost exception is the RuntimeException that wraps anything comes
from splitSerializer.close()
+ Assert.assertEquals(RuntimeException.class, e.getClass());
+
+ // wrapperRuntimeException wraps anything thrown from
splitSerializer.writeSplit()
+ Throwable wrapperRuntimeException = e.getCause();
+ Assert.assertEquals(RuntimeException.class,
wrapperRuntimeException.getClass());
+
+ Throwable originalException = wrapperRuntimeException.getCause();
+ Assert.assertEquals(IOException.class, originalException.getClass());
+ Assert
+
.assertTrue(originalException.getMessage().contains(HiveSplitGeneratorSerializerException.EXCEPTION_MESSAGE));
+
+ Assert.assertTrue("Already running future in not supposed to be
cancelled with the current implementation",
+ splitGenerator.split0Finished.get());
+ Assert.assertFalse("A future started after a failure is not supposed to
run at all",
+ splitGenerator.split2Finished.get());
+ }
+ }
+
+ private InputSplit[] getSplits(int numSplits) {
+ InputSplit[] splits = new InputSplit[numSplits];
+
+ for (int i = 0; i < numSplits; i++) {
+ splits[i] =
+ new HiveInputSplit(new NullRowsInputFormat.DummyInputSplit("/fake" +
i), ZeroRowsInputFormat.class.getName());
+ }
+ return splits;
+ }
+
+ private String expectedSerializedPathFormat(int count) {
+ return String.format(PATH_FORMAT, QUERY_ID, VERTEX_ID, INPUT_NAME, count);
+ }
+
+ private List<Event> generateEvents(JobConf conf, InputSplit[] splits) throws
Exception {
+ return generateEvents(conf, splits, null);
+ }
+
+ private List<Event> generateEvents(JobConf conf, InputSplit[] splits,
HiveSplitGenerator splitGenerator)
+ throws Exception {
+ if (splitGenerator == null) {
+ InputInitializerContext inputInitializerContext =
getInputInitializerContext(conf);
+ splitGenerator = new HiveSplitGenerator(inputInitializerContext);
+ splitGenerator.prepare(inputInitializerContext);
+ }
+
+ InputSplitInfoMem splitInfo = new InputSplitInfoMem(splits, null, 1, null,
conf);
+ return splitGenerator.createEventList(true, splitInfo);
+ }
+
+ private InputInitializerContext getInputInitializerContext(JobConf conf) {
+ HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "tez");
+ conf.set("_hive.hdfs.session.path", "/tmp");
+ conf.set("_hive.local.session.path", "/tmp");
+
+ HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_QUERY_ID, QUERY_ID);
+
+ MapWork mapWork = new MapWork("Map1");
+ mapWork.deriveLlap(conf, false);
+ HiveConf.setVar(conf, HiveConf.ConfVars.PLAN, "file:/tmp");
+ Utilities.setMapWork(conf, mapWork);
+
+ DataSourceDescriptor dataSource = MRInput.createConfigBuilder(conf,
HiveInputFormat.class).build();
+ UserPayload userPayload = dataSource.getInputDescriptor().getUserPayload();
+ return new InputInitializerContextForTest(userPayload, conf);
+ }
+
+ public static class HiveSplitGeneratorSerializerException extends
HiveSplitGenerator {
+ private static final String EXCEPTION_MESSAGE = "Cannot write file to
path";
+ private final AtomicBoolean split0Finished = new AtomicBoolean(false);
+ private final AtomicBoolean split2Finished = new AtomicBoolean(false);
+
+ class SplitSerializerWithException extends SplitSerializer {
+ SplitSerializerWithException() throws IOException {
+ super();
+ }
+
+ @Override
+ InputDataInformationEvent write(int count, MRSplitProto mrSplit) {
+ // mimic the following scenario: split #2 starts to be processed after
split #1 failed,
+ // so it won't run at all due to anyTaskFailed check
+ if (count == 2) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return super.write(count, mrSplit);
+ }
+
+ @Override
+ void writeSplit(int count, MRSplitProto mrSplit, Path filePath) throws
IOException {
+ // first split write takes longer time, started before the failing task
+ // current implementation of the waitFor doesn't cancel it
+ LOG.info("Write split #{}", count);
+ if (count == 0) {
+ try {
+ Thread.sleep(1000);
+ split0Finished.set(true);
+ LOG.info("Split #0 finished");
+ } catch (InterruptedException e) {
+ LOG.info("Split #0 catches InterruptedException, this is not
supposed to happen");
+ throw new IOException(e);
+ }
+ }
+ // writing second split fails
+ if (count == 1) {
+ LOG.info("Split #1 is about to throw exception");
+ throw new IOException(EXCEPTION_MESSAGE + ": " + filePath);
+ }
+
+ // we're not supposed to reach this due to anyTaskFailed check
+ if (count == 2) {
+ split2Finished.set(true);
+ LOG.info("Split #2 finished");
+ }
+ }
+ }
+
+ public HiveSplitGeneratorSerializerException(InputInitializerContext
initializerContext) {
+ super(initializerContext);
+ }
+
+ @Override
+ SplitSerializer getSplitSerializer() throws IOException {
+ return new SplitSerializerWithException();
+ }
+ }
+
+ public static class InputInitializerContextForTest implements
InputInitializerContext {
+
+ private final UserPayload payload;
+ private final Configuration vertexConfig;
+
+ public InputInitializerContextForTest(UserPayload payload, Configuration
vertexConfig) {
+ this.payload = payload;
+ this.vertexConfig = vertexConfig;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ return APP_ID;
+ }
+
+ @Override
+ public String getDAGName() {
+ return "FakeDAG";
+ }
+
+ @Override
+ public Configuration getVertexConfiguration() {
+ return vertexConfig;
+ }
+
+ @Override
+ public String getInputName() {
+ return INPUT_NAME;
+ }
+
+ @Override
+ public UserPayload getInputUserPayload() {
+ return payload;
+ }
+
+ @Override
+ public int getNumTasks() {
+ return 100;
+ }
+
+ @Override
+ public Resource getVertexTaskResource() {
+ return Resource.newInstance(1024, 1);
+ }
+
+ @Override
+ public int getVertexId() {
+ return VERTEX_ID;
+ }
+
+ @Override
+ public Resource getTotalAvailableResource() {
+ return Resource.newInstance(10240, 10);
+ }
+
+ @Override
+ public int getNumClusterNodes() {
+ return 10;
+ }
+
+ @Override
+ public int getDAGAttemptNumber() {
+ return 1;
+ }
+
+ @Override
+ public int getVertexNumTasks(String vertexName) {
+ throw new UnsupportedOperationException("getVertexNumTasks not
implemented in this mock");
+ }
+
+ @Override
+ public void registerForVertexStateUpdates(String vertexName,
Set<VertexState> stateSet) {
+ throw new UnsupportedOperationException("getVertexNumTasks not
implemented in this mock");
+ }
+
+ @Override
+ public void addCounters(TezCounters tezCounters) {
+ throw new UnsupportedOperationException("addCounters not implemented in
this mock");
+ }
+
+ @Override
+ public UserPayload getUserPayload() {
+ throw new UnsupportedOperationException("getUserPayload not implemented
in this mock");
+ }
+
+ }
+}
diff --git a/ql/src/test/queries/clientpositive/tez_split_serialization.q
b/ql/src/test/queries/clientpositive/tez_split_serialization.q
new file mode 100644
index 00000000000..8d59428595e
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez_split_serialization.q
@@ -0,0 +1,14 @@
+set hive.fetch.task.conversion = none;
+
+create external table test_table(
+ strcol string,
+ intcol integer
+) stored as ORC;
+
+insert into table test_table values ('ABC', 1), ('DEF', 2);
+select * from test_table;
+
+-- split serialization can be sanity-checked from hive.log by
HiveSplitGenerator messages
+-- 10 bytes threshold must trigger split serialization to filesystem
+set hive.tez.input.fs.serialization.threshold = 10;
+select * from test_table;
diff --git
a/ql/src/test/results/clientpositive/llap/tez_split_serialization.q.out
b/ql/src/test/results/clientpositive/llap/tez_split_serialization.q.out
new file mode 100644
index 00000000000..8167d6f423e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/tez_split_serialization.q.out
@@ -0,0 +1,44 @@
+PREHOOK: query: create external table test_table(
+ strcol string,
+ intcol integer
+) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_table
+POSTHOOK: query: create external table test_table(
+ strcol string,
+ intcol integer
+) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_table
+PREHOOK: query: insert into table test_table values ('ABC', 1), ('DEF', 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_table
+POSTHOOK: query: insert into table test_table values ('ABC', 1), ('DEF', 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_table
+POSTHOOK: Lineage: test_table.intcol SCRIPT []
+POSTHOOK: Lineage: test_table.strcol SCRIPT []
+PREHOOK: query: select * from test_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_table
+#### A masked pattern was here ####
+ABC 1
+DEF 2
+PREHOOK: query: select * from test_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_table
+#### A masked pattern was here ####
+ABC 1
+DEF 2