This is an automated email from the ASF dual-hosted git repository.

abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 3aa05c84e40 HIVE-28165: HiveSplitGenerator: send splits through 
filesystem instead of RPC in case of big payload (#5449) (Laszlo Bodor reviewed 
by Denys Kuzmenko)
3aa05c84e40 is described below

commit 3aa05c84e401cf4b67cf17101233621e77bd7325
Author: Bodor Laszlo <[email protected]>
AuthorDate: Wed Sep 18 10:53:20 2024 +0200

    HIVE-28165: HiveSplitGenerator: send splits through filesystem instead of 
RPC in case of big payload (#5449) (Laszlo Bodor reviewed by Denys Kuzmenko)
---
 .../java/org/apache/hadoop/hive/conf/HiveConf.java |   7 +
 .../hive/ql/exec/tez/HiveSplitGenerator.java       | 173 +++++++++-
 .../hive/ql/udf/generic/GenericUDTFGetSplits.java  |   8 +-
 .../hive/ql/exec/tez/TestHiveSplitGenerator.java   | 351 +++++++++++++++++++++
 .../clientpositive/tez_split_serialization.q       |  14 +
 .../llap/tez_split_serialization.q.out             |  44 +++
 6 files changed, 587 insertions(+), 10 deletions(-)

diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java 
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 280f9a19116..5252ec8724b 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4778,6 +4778,13 @@ public class HiveConf extends Configuration {
         "Class to use for calculating available slots during split 
generation"),
     
HIVE_TEZ_GENERATE_CONSISTENT_SPLITS("hive.tez.input.generate.consistent.splits",
 true,
         "Whether to generate consistent split locations when generating splits 
in the AM"),
+    
HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD("hive.tez.input.fs.serialization.threshold",
 268435456,
+        "When the cummulative size of the splits is larger than this (in 
bytes), HiveSplitGenerator"
+            + " will start to serialize splits to tez scratchdir instead of 
being sent as RPC payloads directly."
+            + " Default is 256MB."
+            + "-1 disables this feature."),
+    
HIVE_TEZ_INPUT_FS_SERIALIZATION_THREADS("hive.tez.input.fs.serialization.threads",
 8,
+        "Number of threads used for serializing split payloads to 
filesystem."),
     HIVE_PREWARM_ENABLED("hive.prewarm.enabled", false, "Enables container 
prewarm for Tez(Hadoop 2 only)"),
     HIVE_PREWARM_NUM_CONTAINERS("hive.prewarm.numcontainers", 10, "Controls 
the number of containers to prewarm for Tez (Hadoop 2 only)"),
     HIVE_STAGE_ID_REARRANGE("hive.stageid.rearrange", "none", new 
StringSet("none", "idonly", "traverse", "execution"), ""),
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java 
b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
index 46df46bbdf6..d11f135ccb3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
@@ -22,15 +22,30 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
+import java.util.Queue;
 import java.util.Set;
-
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import com.google.protobuf.ByteString;
+import java.nio.ByteBuffer;
 
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -56,6 +71,8 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.split.SplitLocationProvider;
 import org.apache.hadoop.mapreduce.split.TezMapReduceSplitsGrouper;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.Time;
+import org.apache.tez.common.TezCommonUtils;
 import org.apache.tez.common.TezUtils;
 import org.apache.tez.dag.api.TaskLocationHint;
 import org.apache.tez.dag.api.VertexLocationHint;
@@ -75,6 +92,7 @@ import 
org.apache.tez.runtime.api.events.InputInitializerEvent;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
 /**
  * This class is used to generate splits inside the AM on the cluster. It
@@ -136,12 +154,13 @@ public class HiveSplitGenerator extends InputInitializer {
     this.numSplits = Optional.empty();
   }
 
-  private void prepare(InputInitializerContext initializerContext) throws 
IOException, SerDeException {
+  @VisibleForTesting
+  void prepare(InputInitializerContext initializerContext) throws IOException, 
SerDeException {
     userPayloadProto =
         
MRInputHelpers.parseMRInputPayload(initializerContext.getInputUserPayload());
 
     this.conf = 
TezUtils.createConfFromByteString(userPayloadProto.getConfigurationBytes());
-    
+
     this.jobConf = new JobConf(conf);
 
     // Read all credentials into the credentials instance stored in JobConf.
@@ -154,6 +173,102 @@ public class HiveSplitGenerator extends InputInitializer {
     LOG.info("SplitLocationProvider: " + splitLocationProvider);
   }
 
+  /**
+   * SplitSerializer is a helper class for taking care of serializing splits 
to the tez scratch dir
+   * when a size criteria defined by 
"hive.tez.input.fs.serialization.threshold" is met.
+   * It utilizes an ExecutorService for parallel writes to prevent a single 
split write operation
+   * becoming the bottleneck (as write() is called from a loop currently).
+   */
+  class SplitSerializer implements AutoCloseable {
+    private static final String FILE_PATH_FORMAT = 
"%s/events/%s/%d_%s_InputDataInformationEvent_%d";
+
+    // fields needed for filepath
+    private String queryId;
+    private String inputName;
+    private int vertexId;
+    private Path appStagingPath;
+    // filesystem and executor
+    private FileSystem fs;
+    private ExecutorService executor;
+
+    private AtomicBoolean anyTaskFailed;
+    private List<Future<?>> asyncTasks;
+
+    @VisibleForTesting
+    SplitSerializer() throws IOException {
+      if (getContext() == null) {
+        // this typically happens in case the split generation is not in the 
Tez AM
+        // in that case we're not interested in this feature, so it's fine to 
return here and fail later
+        return;
+      }
+      queryId = jobConf.get(HiveConf.ConfVars.HIVE_QUERY_ID.varname);
+      inputName = getContext().getInputName();
+      vertexId = getContext().getVertexId();
+      appStagingPath = TezCommonUtils.getTezSystemStagingPath(jobConf, 
getContext().getApplicationId().toString());
+
+      fs = appStagingPath.getFileSystem(jobConf);
+
+      int numThreads = HiveConf.getIntVar(jobConf, 
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THREADS);
+      executor = Executors.newFixedThreadPool(numThreads,
+          new 
ThreadFactoryBuilder().setNameFormat("HiveSplitGenerator.SplitSerializer Thread 
- " + "#%d").build());
+      anyTaskFailed = new AtomicBoolean(false);
+      asyncTasks = new ArrayList<>();
+    }
+
+    @VisibleForTesting
+    InputDataInformationEvent write(int count, MRSplitProto mrSplit) {
+      InputDataInformationEvent diEvent;
+      Path filePath = getSerializedFilePath(count);
+
+      Runnable task = () -> {
+        if (!anyTaskFailed.get()) {
+          try {
+            writeSplit(count, mrSplit, filePath);
+          } catch (IOException e) {
+            anyTaskFailed.set(true);
+            throw new RuntimeException(e);
+          }
+        }
+      };
+      asyncTasks.add(CompletableFuture.runAsync(task, executor));
+      return InputDataInformationEvent.createWithSerializedPath(count, 
filePath.toString());
+    }
+
+    @VisibleForTesting
+    void writeSplit(int count, MRSplitProto mrSplit, Path filePath) throws 
IOException {
+      long fileWriteStarted = Time.monotonicNow();
+      try (FSDataOutputStream out = fs.create(filePath, false)) {
+        mrSplit.writeTo(out);
+      }
+      LOG.debug("Split #{} event to output path: {} written in {} ms", count, 
filePath,
+          fileWriteStarted - Time.monotonicNow());
+    }
+
+    Path getSerializedFilePath(int index) {
+      // e.g. staging_dir/events/queryid/inputtable_InputDataInformationEvent_0
+      return new Path(String.format(FILE_PATH_FORMAT, appStagingPath, queryId, 
vertexId, inputName, index));
+    }
+
+    @Override
+    public void close() {
+      try {
+        if (asyncTasks != null) {
+          CompletableFuture.allOf(asyncTasks.toArray(new 
CompletableFuture[0])).get();
+        }
+      } catch (InterruptedException e) {
+        LOG.info("Interrupted while generating splits", e);
+        Thread.currentThread().interrupt();
+      } catch (ExecutionException e) {// ExecutionException wraps the original 
exception
+        LOG.error("Exception while generating splits", e.getCause());
+        throw new RuntimeException(e.getCause());
+      } finally {
+        if (executor != null) {
+          executor.shutdown();
+        }
+      }
+    }
+  }
+
   @SuppressWarnings("unchecked")
   @Override
   public List<Event> initialize() throws Exception {
@@ -351,7 +466,9 @@ public class HiveSplitGenerator extends InputInitializer {
     return splits;
   }
 
-  private List<Event> createEventList(boolean sendSerializedEvents, 
InputSplitInfoMem inputSplitInfo) {
+  @VisibleForTesting
+  List<Event> createEventList(boolean sendSerializedEvents, InputSplitInfoMem 
inputSplitInfo)
+      throws IOException {
 
     List<Event> events = 
Lists.newArrayListWithCapacity(inputSplitInfo.getNumTasks() + 1);
 
@@ -362,13 +479,46 @@ public class HiveSplitGenerator extends InputInitializer {
     events.add(configureVertexEvent);
 
     if (sendSerializedEvents) {
-      MRSplitsProto splitsProto = inputSplitInfo.getSplitsProto();
       int count = 0;
-      for (MRSplitProto mrSplit : splitsProto.getSplitsList()) {
-        InputDataInformationEvent diEvent = 
InputDataInformationEvent.createWithSerializedPayload(
-            count++, mrSplit.toByteString().asReadOnlyByteBuffer());
-        events.add(diEvent);
+      long inMemoryPayloadSize = 0;
+      long serializedPayloadSize = 0;
+
+      int payloadSerializationThresholdBytes =
+          HiveConf.getIntVar(jobConf, 
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD);
+
+      List<MRSplitProto> splits = 
inputSplitInfo.getSplitsProto().getSplitsList();
+      LOG.debug("Start creating events for {} splits", splits.size());
+      long startTime = Time.monotonicNow();
+
+      try (SplitSerializer splitSerializer = getSplitSerializer()) {
+        for (MRSplitProto mrSplit : splits) {
+          ByteBuffer payloadBuffer = 
mrSplit.toByteString().asReadOnlyByteBuffer();
+          int payloadSize = payloadBuffer.limit();
+          boolean shouldSerializeEventToFile =
+              payloadSerializationThresholdBytes != -1 && inMemoryPayloadSize 
>= payloadSerializationThresholdBytes;
+          LOG.debug("Split #{}, byteBuffer size: {} bytes", count, 
payloadSize);
+
+          InputDataInformationEvent diEvent = null;
+
+          if (shouldSerializeEventToFile) {
+            LOG.debug("Serialize to path: {}, current in-memory payload: {} 
bytes >= threshold: {} bytes",
+                splitSerializer.getSerializedFilePath(count), 
inMemoryPayloadSize, payloadSerializationThresholdBytes);
+            serializedPayloadSize += payloadSize;
+            diEvent = splitSerializer.write(count, mrSplit);
+          } else {
+            inMemoryPayloadSize += payloadSize;
+            diEvent = 
InputDataInformationEvent.createWithSerializedPayload(count, payloadBuffer);
+          }
+          events.add(diEvent);
+          count += 1;
+        }
       }
+
+      // this is useful for making decisions regarding split serialization 
threshold
+      long elapsed = Time.monotonicNow() - startTime;
+      LOG.info(
+          "Finished creating events ({} splits) in {} ms, size of payloads: in 
memory: {} bytes, serialized to fs: {} bytes",
+          splits.size(), elapsed, inMemoryPayloadSize, serializedPayloadSize);
     } else {
       int count = 0;
       for (org.apache.hadoop.mapred.InputSplit split : 
inputSplitInfo.getOldFormatSplits()) {
@@ -380,6 +530,11 @@ public class HiveSplitGenerator extends InputInitializer {
     return events;
   }
 
+  @VisibleForTesting
+  SplitSerializer getSplitSerializer() throws IOException {
+    return new SplitSerializer();
+  }
+
   @Override
   public void onVertexStateUpdated(VertexStateUpdate stateUpdate) {
     // pruner registers for vertex state updates after it is ready to handle 
them
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
index b1c3b767dec..bb029442d66 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java
@@ -465,7 +465,13 @@ public class GenericUDTFGetSplits extends GenericUDTF {
       Preconditions.checkState(HiveConf.getBoolVar(wxConf,
               ConfVars.LLAP_CLIENT_CONSISTENT_SPLITS));
 
-      HiveSplitGenerator splitGenerator = new HiveSplitGenerator(wxConf, 
mapWork, false, inputArgNumSplits);
+      // we're not interested in split fs serialization optimization in this 
case
+      // it was implemented for split generation in TezAM
+      // this can be removed any time when it turns out that's needed here too
+      HiveConf.setIntVar(wxConf, 
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, -1);
+
+      HiveSplitGenerator splitGenerator =
+          new HiveSplitGenerator(wxConf, mapWork, false, inputArgNumSplits);
       List<Event> eventList = splitGenerator.initialize();
       int numGroupedSplitsGenerated = eventList.size() - 1;
       InputSplit[] result = new InputSplit[numGroupedSplitsGenerated];
diff --git 
a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHiveSplitGenerator.java 
b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHiveSplitGenerator.java
new file mode 100644
index 00000000000..8780cb67cd4
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHiveSplitGenerator.java
@@ -0,0 +1,351 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.exec.tez;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.exec.tez.HiveSplitGenerator.SplitSerializer;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat.HiveInputSplit;
+import org.apache.hadoop.hive.ql.io.NullRowsInputFormat;
+import org.apache.hadoop.hive.ql.io.ZeroRowsInputFormat;
+import org.apache.hadoop.hive.ql.plan.MapWork;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.tez.common.counters.TezCounters;
+import org.apache.tez.dag.api.DataSourceDescriptor;
+import org.apache.tez.dag.api.UserPayload;
+import org.apache.tez.dag.api.event.VertexState;
+import org.apache.tez.mapreduce.hadoop.InputSplitInfoMem;
+import org.apache.tez.mapreduce.input.MRInput;
+import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto;
+import org.apache.tez.runtime.api.Event;
+import org.apache.tez.runtime.api.InputInitializerContext;
+import org.apache.tez.runtime.api.events.InputDataInformationEvent;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestHiveSplitGenerator {
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(TestHiveSplitGenerator.class);
+
+  private static final String QUERY_ID = "hive_" + System.currentTimeMillis();
+  private static final ApplicationId APP_ID = ApplicationId.newInstance(1000, 
200);
+  private static final String INPUT_NAME = "MRInput";
+  private static final int VERTEX_ID = 0;
+  private static final String PATH_FORMAT = 
"events/%s/%d_%s_InputDataInformationEvent_%d";
+
+  @Test
+  public void testSplitSerializationUsingFileSystem() throws Exception {
+    JobConf conf = new JobConf(new HiveConf());
+    // 0 bytes will force a split serialization to filesystem
+    HiveConf.setIntVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, 0);
+
+    InputSplit[] splits = getSplits(1);
+    String serializedPath = ((InputDataInformationEvent) generateEvents(conf, 
splits).get(1)).getSerializedPath();
+    // second split is serialized
+    Assert.assertTrue(
+        "Serialized path should contain a pattern like "
+            + 
"'events/$queryid/$vertexid_$inputname_InputDataInformationEvent_$count', but 
found: " + serializedPath,
+        serializedPath.contains(expectedSerializedPathFormat(0)));
+  }
+
+  @Test
+  public void testSplitSerializationInMemoryLimitExceeded() throws Exception {
+    JobConf conf = new JobConf(new HiveConf());
+    // split size is around 200 bytes, so first one will be held in memory, 
second will be serialized (200 > 10)
+    // this is the most typical usecase of the split fs serialization feature
+    HiveConf.setIntVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, 10);
+
+    InputSplit[] splits = getSplits(2);
+    List<Event> events = generateEvents(conf, splits);
+    String serializedPath1 = ((InputDataInformationEvent) 
events.get(1)).getSerializedPath();
+    String serializedPath2 = ((InputDataInformationEvent) 
events.get(2)).getSerializedPath();
+
+    // first split is held in memory, no serialized path found
+    Assert.assertNull(serializedPath1);
+
+    // second split is serialized
+    Assert.assertTrue(
+        "Serialized path should contain a pattern like "
+            + 
"'events/$queryid/$vertexid_$inputname_InputDataInformationEvent_$count', but 
found: " + serializedPath2,
+        serializedPath2.contains(expectedSerializedPathFormat(1)));
+  }
+
+  @Test
+  public void testSplitSerializationNoFileSystemBecauseOfDefaultConfig() 
throws Exception {
+    JobConf conf = new JobConf(new HiveConf());
+
+    InputSplit[] splits = getSplits(1);
+    String serializedPath = ((InputDataInformationEvent) generateEvents(conf, 
splits).get(1)).getSerializedPath();
+    // the default value 512KB is to large to trigger split serialization to 
filesystem
+    Assert.assertNull(serializedPath);
+  }
+
+  @Test
+  public void testSplitSerializationDisabledFilesystem() throws Exception {
+    JobConf conf = new JobConf(new HiveConf());
+    // -1 disables the split serialization to filesystem
+    HiveConf.setIntVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, -1);
+
+    InputSplit[] splits = getSplits(1);
+    String serializedPath = ((InputDataInformationEvent) generateEvents(conf, 
splits).get(1)).getSerializedPath();
+    Assert.assertNull(serializedPath);
+  }
+
+  @Test
+  public void testExceptionIsPropagatedFromSplitSerializer() throws Exception {
+    JobConf conf = new JobConf(new HiveConf());
+    HiveConf.setIntVar(conf, 
HiveConf.ConfVars.HIVE_TEZ_INPUT_FS_SERIALIZATION_THRESHOLD, 0);
+
+    InputSplit[] splits = getSplits(3);
+    InputInitializerContext context = getInputInitializerContext(conf);
+    HiveSplitGeneratorSerializerException splitGenerator = new 
HiveSplitGeneratorSerializerException(context);
+    splitGenerator.prepare(context);
+    try {
+      String serializedPath =
+          ((InputDataInformationEvent) generateEvents(conf, splits, 
splitGenerator).get(1)).getSerializedPath();
+      Assert.fail("HiveSplitGeneratorSerializerException should fail");
+    } catch (Exception e) {
+      // outermost exception is the RuntimeException that wraps anything comes 
from splitSerializer.close()
+      Assert.assertEquals(RuntimeException.class, e.getClass());
+
+      // wrapperRuntimeException wraps anything thrown from 
splitSerializer.writeSplit()
+      Throwable wrapperRuntimeException = e.getCause();
+      Assert.assertEquals(RuntimeException.class, 
wrapperRuntimeException.getClass());
+
+      Throwable originalException = wrapperRuntimeException.getCause();
+      Assert.assertEquals(IOException.class, originalException.getClass());
+      Assert
+          
.assertTrue(originalException.getMessage().contains(HiveSplitGeneratorSerializerException.EXCEPTION_MESSAGE));
+
+      Assert.assertTrue("Already running future in not supposed to be 
cancelled with the current implementation",
+          splitGenerator.split0Finished.get());
+      Assert.assertFalse("A future started after a failure is not supposed to 
run at all",
+          splitGenerator.split2Finished.get());
+    }
+  }
+
+  private InputSplit[] getSplits(int numSplits) {
+    InputSplit[] splits = new InputSplit[numSplits];
+
+    for (int i = 0; i < numSplits; i++) {
+      splits[i] =
+          new HiveInputSplit(new NullRowsInputFormat.DummyInputSplit("/fake" + 
i), ZeroRowsInputFormat.class.getName());
+    }
+    return splits;
+  }
+
+  private String expectedSerializedPathFormat(int count) {
+    return String.format(PATH_FORMAT, QUERY_ID, VERTEX_ID, INPUT_NAME, count);
+  }
+
+  private List<Event> generateEvents(JobConf conf, InputSplit[] splits) throws 
Exception {
+    return generateEvents(conf, splits, null);
+  }
+
+  private List<Event> generateEvents(JobConf conf, InputSplit[] splits, 
HiveSplitGenerator splitGenerator)
+      throws Exception {
+    if (splitGenerator == null) {
+      InputInitializerContext inputInitializerContext = 
getInputInitializerContext(conf);
+      splitGenerator = new HiveSplitGenerator(inputInitializerContext);
+      splitGenerator.prepare(inputInitializerContext);
+    }
+
+    InputSplitInfoMem splitInfo = new InputSplitInfoMem(splits, null, 1, null, 
conf);
+    return splitGenerator.createEventList(true, splitInfo);
+  }
+
+  private InputInitializerContext getInputInitializerContext(JobConf conf) {
+    HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "tez");
+    conf.set("_hive.hdfs.session.path", "/tmp");
+    conf.set("_hive.local.session.path", "/tmp");
+
+    HiveConf.setVar(conf, HiveConf.ConfVars.HIVE_QUERY_ID, QUERY_ID);
+
+    MapWork mapWork = new MapWork("Map1");
+    mapWork.deriveLlap(conf, false);
+    HiveConf.setVar(conf, HiveConf.ConfVars.PLAN, "file:/tmp");
+    Utilities.setMapWork(conf, mapWork);
+
+    DataSourceDescriptor dataSource = MRInput.createConfigBuilder(conf, 
HiveInputFormat.class).build();
+    UserPayload userPayload = dataSource.getInputDescriptor().getUserPayload();
+    return new InputInitializerContextForTest(userPayload, conf);
+  }
+
+  public static class HiveSplitGeneratorSerializerException extends 
HiveSplitGenerator {
+    private static final String EXCEPTION_MESSAGE = "Cannot write file to 
path";
+    private final AtomicBoolean split0Finished = new AtomicBoolean(false);
+    private final AtomicBoolean split2Finished = new AtomicBoolean(false);
+
+    class SplitSerializerWithException extends SplitSerializer {
+      SplitSerializerWithException() throws IOException {
+        super();
+      }
+
+      @Override
+      InputDataInformationEvent write(int count, MRSplitProto mrSplit) {
+        // mimic the following scenario: split #2 starts to be processed after 
split #1 failed,
+        // so it won't run at all due to anyTaskFailed check
+        if (count == 2) {
+          try {
+            Thread.sleep(1000);
+          } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+          }
+        }
+        return super.write(count, mrSplit);
+      }
+
+      @Override
+      void writeSplit(int count, MRSplitProto mrSplit, Path filePath) throws 
IOException {
+        // first split write takes longer time, started before the failing task
+        // current implementation of the waitFor doesn't cancel it
+        LOG.info("Write split #{}", count);
+        if (count == 0) {
+          try {
+            Thread.sleep(1000);
+            split0Finished.set(true);
+            LOG.info("Split #0 finished");
+          } catch (InterruptedException e) {
+            LOG.info("Split #0 catches InterruptedException, this is not 
supposed to happen");
+            throw new IOException(e);
+          }
+        }
+        // writing second split fails
+        if (count == 1) {
+          LOG.info("Split #1 is about to throw exception");
+          throw new IOException(EXCEPTION_MESSAGE + ": " + filePath);
+        }
+
+        // we're not supposed to reach this due to anyTaskFailed check
+        if (count == 2) {
+          split2Finished.set(true);
+          LOG.info("Split #2 finished");
+        }
+      }
+    }
+
+    public HiveSplitGeneratorSerializerException(InputInitializerContext 
initializerContext) {
+      super(initializerContext);
+    }
+
+    @Override
+    SplitSerializer getSplitSerializer() throws IOException {
+      return new SplitSerializerWithException();
+    }
+  }
+
+  public static class InputInitializerContextForTest implements 
InputInitializerContext {
+
+    private final UserPayload payload;
+    private final Configuration vertexConfig;
+
+    public InputInitializerContextForTest(UserPayload payload, Configuration 
vertexConfig) {
+      this.payload = payload;
+      this.vertexConfig = vertexConfig;
+    }
+
+    @Override
+    public ApplicationId getApplicationId() {
+      return APP_ID;
+    }
+
+    @Override
+    public String getDAGName() {
+      return "FakeDAG";
+    }
+
+    @Override
+    public Configuration getVertexConfiguration() {
+      return vertexConfig;
+    }
+
+    @Override
+    public String getInputName() {
+      return INPUT_NAME;
+    }
+
+    @Override
+    public UserPayload getInputUserPayload() {
+      return payload;
+    }
+
+    @Override
+    public int getNumTasks() {
+      return 100;
+    }
+
+    @Override
+    public Resource getVertexTaskResource() {
+      return Resource.newInstance(1024, 1);
+    }
+
+    @Override
+    public int getVertexId() {
+      return VERTEX_ID;
+    }
+
+    @Override
+    public Resource getTotalAvailableResource() {
+      return Resource.newInstance(10240, 10);
+    }
+
+    @Override
+    public int getNumClusterNodes() {
+      return 10;
+    }
+
+    @Override
+    public int getDAGAttemptNumber() {
+      return 1;
+    }
+
+    @Override
+    public int getVertexNumTasks(String vertexName) {
+      throw new UnsupportedOperationException("getVertexNumTasks not 
implemented in this mock");
+    }
+
+    @Override
+    public void registerForVertexStateUpdates(String vertexName, 
Set<VertexState> stateSet) {
+      throw new UnsupportedOperationException("getVertexNumTasks not 
implemented in this mock");
+    }
+
+    @Override
+    public void addCounters(TezCounters tezCounters) {
+      throw new UnsupportedOperationException("addCounters not implemented in 
this mock");
+    }
+
+    @Override
+    public UserPayload getUserPayload() {
+      throw new UnsupportedOperationException("getUserPayload not implemented 
in this mock");
+    }
+
+  }
+}
diff --git a/ql/src/test/queries/clientpositive/tez_split_serialization.q 
b/ql/src/test/queries/clientpositive/tez_split_serialization.q
new file mode 100644
index 00000000000..8d59428595e
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/tez_split_serialization.q
@@ -0,0 +1,14 @@
+set hive.fetch.task.conversion = none;
+
+create external table test_table(
+  strcol string,
+  intcol integer
+) stored as ORC;
+
+insert into table test_table values ('ABC', 1), ('DEF', 2);
+select * from test_table;
+
+-- split serialization can be sanity-checked from hive.log by 
HiveSplitGenerator messages
+-- 10 bytes threshold must trigger split serialization to filesystem
+set hive.tez.input.fs.serialization.threshold = 10;
+select * from test_table;
diff --git 
a/ql/src/test/results/clientpositive/llap/tez_split_serialization.q.out 
b/ql/src/test/results/clientpositive/llap/tez_split_serialization.q.out
new file mode 100644
index 00000000000..8167d6f423e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/tez_split_serialization.q.out
@@ -0,0 +1,44 @@
+PREHOOK: query: create external table test_table(
+  strcol string,
+  intcol integer
+) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_table
+POSTHOOK: query: create external table test_table(
+  strcol string,
+  intcol integer
+) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_table
+PREHOOK: query: insert into table test_table values ('ABC', 1), ('DEF', 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_table
+POSTHOOK: query: insert into table test_table values ('ABC', 1), ('DEF', 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_table
+POSTHOOK: Lineage: test_table.intcol SCRIPT []
+POSTHOOK: Lineage: test_table.strcol SCRIPT []
+PREHOOK: query: select * from test_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_table
+#### A masked pattern was here ####
+ABC    1
+DEF    2
+PREHOOK: query: select * from test_table
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_table
+#### A masked pattern was here ####
+POSTHOOK: query: select * from test_table
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_table
+#### A masked pattern was here ####
+ABC    1
+DEF    2

Reply via email to