Revert "HIVE-13507. Improved logging for ptest. (Siddharth Seth, reviewed by Thejas M Nair)"
This patch is causing Hive ptest to stay in a loop creating nodes due to an error exception. This reverts commit fcc2e79511bb1a0db70f4a17ede6ade5e73b1f42. Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9b5eb45b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9b5eb45b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9b5eb45b Branch: refs/heads/llap Commit: 9b5eb45b83d29adf0eb83f65b8821384d9254985 Parents: 6c3d3b4 Author: Sergio Pena <sergio.p...@cloudera.com> Authored: Wed Apr 20 13:36:31 2016 -0500 Committer: Sergio Pena <sergio.p...@cloudera.com> Committed: Wed Apr 20 13:36:31 2016 -0500 ---------------------------------------------------------------------- .../hive/ptest/execution/ExecutionPhase.java | 2 - .../hive/ptest/execution/HostExecutor.java | 48 ++------------------ .../hive/ptest/execution/LocalCommand.java | 18 +------- .../apache/hive/ptest/execution/PrepPhase.java | 1 - .../apache/hive/ptest/execution/conf/Host.java | 3 -- 5 files changed, 7 insertions(+), 65 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java index 6063afc..3026ea0 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/ExecutionPhase.java @@ -86,8 +86,6 @@ public class ExecutionPhase extends Phase { isolatedWorkQueue.add(batch); } } - logger.info("ParallelWorkQueueSize={}, IsolatedWorkQueueSize={}", parallelWorkQueue.size(), - isolatedWorkQueue.size()); try { int expectedNumHosts = hostExecutors.size(); initalizeHosts(); http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java index 735b261..b05d2c2 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/HostExecutor.java @@ -29,7 +29,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; -import com.google.common.base.Stopwatch; import org.apache.hive.ptest.execution.conf.Host; import org.apache.hive.ptest.execution.conf.TestBatch; import org.apache.hive.ptest.execution.ssh.RSyncCommand; @@ -66,8 +65,6 @@ class HostExecutor { private final File mFailedTestLogDir; private final long mNumPollSeconds; private volatile boolean mShutdown; - private int numParallelBatchesProcessed = 0; - private int numIsolatedBatchesProcessed = 0; HostExecutor(Host host, String privateKey, ListeningExecutorService executor, SSHCommandExecutor sshCommandExecutor, @@ -103,18 +100,7 @@ class HostExecutor { return mExecutor.submit(new Callable<Void>() { @Override public Void call() throws Exception { - Stopwatch stopwatch = Stopwatch.createStarted(); - mLogger.info("Starting SubmitTests on host {}", getHost()); - try { - executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults); - } finally { - stopwatch.stop(); - mLogger.info("Finishing submitTests on host: {}. ElapsedTime(seconds)={}," + - " NumParallelBatchesProcessed={}, NumIsolatedBatchesProcessed={}", - new Object[]{getHost().toString(), - stopwatch.elapsed(TimeUnit.SECONDS), numParallelBatchesProcessed, - numIsolatedBatchesProcessed}); - } + executeTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults); return null; } @@ -157,7 +143,6 @@ class HostExecutor { @Override public Void call() throws Exception { TestBatch batch = null; - Stopwatch sw = Stopwatch.createUnstarted(); try { do { batch = parallelWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS); @@ -166,16 +151,8 @@ class HostExecutor { return null; } if(batch != null) { - numParallelBatchesProcessed++; - sw.reset().start(); - try { - if (!executeTestBatch(drone, batch, failedTestResults)) { - failedTestResults.add(batch); - } - } finally { - sw.stop(); - mLogger.info("Finished processing parallel batch [{}] on host {}. ElapsedTime(seconds)={}", - new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)}); + if(!executeTestBatch(drone, batch, failedTestResults)) { + failedTestResults.add(batch); } } } while(!mShutdown && !parallelWorkQueue.isEmpty()); @@ -199,22 +176,12 @@ class HostExecutor { mLogger.info("Starting isolated execution on " + mHost.getName()); for(Drone drone : ImmutableList.copyOf(mDrones)) { TestBatch batch = null; - Stopwatch sw = Stopwatch.createUnstarted(); try { do { - batch = isolatedWorkQueue.poll(mNumPollSeconds, TimeUnit.SECONDS); if(batch != null) { - numIsolatedBatchesProcessed++; - sw.reset().start(); - try { - if (!executeTestBatch(drone, batch, failedTestResults)) { - failedTestResults.add(batch); - } - } finally { - sw.stop(); - mLogger.info("Finished processing isolated batch [{}] on host {}. ElapsedTime(seconds)={}", - new Object[]{batch.getName(), getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)}); + if(!executeTestBatch(drone, batch, failedTestResults)) { + failedTestResults.add(batch); } } } while(!mShutdown && !isolatedWorkQueue.isEmpty()); @@ -248,15 +215,10 @@ class HostExecutor { Templates.writeTemplateResult("batch-exec.vm", script, templateVariables); copyToDroneFromLocal(drone, script.getAbsolutePath(), "$localDir/$instanceName/scratch/" + scriptName); script.delete(); - Stopwatch sw = Stopwatch.createStarted(); mLogger.info(drone + " executing " + batch + " with " + command); RemoteCommandResult sshResult = new SSHCommand(mSSHCommandExecutor, drone.getPrivateKey(), drone.getUser(), drone.getHost(), drone.getInstance(), command, true). call(); - sw.stop(); - mLogger.info("Completed executing tests for batch [{}] on host {}. ElapsedTime(seconds)={}", - new Object[]{batch.getName(), - getHost().toShortString(), sw.elapsed(TimeUnit.SECONDS)}); File batchLogDir = null; if(sshResult.getExitCode() == Constants.EXIT_CODE_UNKNOWN) { throw new AbortDroneException("Drone " + drone.toString() + " exited with " + http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java index 8e2c5c7..ec99656 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/LocalCommand.java @@ -22,28 +22,17 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import com.google.common.base.Stopwatch; import org.slf4j.Logger; public class LocalCommand { - private static final AtomicInteger localCommandCounter = new AtomicInteger(0); - - private final Logger logger; private final Process process; private final StreamReader streamReader; private Integer exitCode; - private final int commandId; - private final Stopwatch stopwatch = Stopwatch.createUnstarted(); public LocalCommand(Logger logger, OutputPolicy outputPolicy, String command) throws IOException { - this.commandId = localCommandCounter.incrementAndGet(); - this.logger = logger; - logger.info("Starting LocalCommandId={}: {}" + commandId, command); - stopwatch.start(); + logger.info("Starting " + command); process = new ProcessBuilder().command(new String[] {"bash", "-c", command}).redirectErrorStream(true).start(); streamReader = new StreamReader(outputPolicy, process.getInputStream()); streamReader.setName("StreamReader-[" + command + "]"); @@ -56,13 +45,10 @@ public class LocalCommand { if(exitCode == null) { exitCode = process.waitFor(); } - stopwatch.stop(); - logger.info("Finished LocalCommandId={}. ElapsedTime(seconds)={}", commandId, stopwatch.elapsed( - TimeUnit.SECONDS)); return exitCode; } } - + public void kill() { synchronized (process) { process.destroy(); http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java index 8fef413..825f0c0 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/PrepPhase.java @@ -62,7 +62,6 @@ public class PrepPhase extends Phase { // source prep start = System.currentTimeMillis(); File sourcePrepScript = new File(mScratchDir, "source-prep.sh"); - logger.info("Writing {} from template", sourcePrepScript); Templates.writeTemplateResult("source-prep.vm", sourcePrepScript, getTemplateDefaults()); execLocally("bash " + sourcePrepScript.getPath()); logger.debug("Deleting " + sourcePrepScript + ": " + sourcePrepScript.delete()); http://git-wip-us.apache.org/repos/asf/hive/blob/9b5eb45b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java ---------------------------------------------------------------------- diff --git a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java index a56824c..c1216c1 100644 --- a/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java +++ b/testutils/ptest2/src/main/java/org/apache/hive/ptest/execution/conf/Host.java @@ -47,9 +47,6 @@ public class Host { public String[] getLocalDirectories() { return localDirectories; } - public String toShortString() { - return name; - } @Override public String toString() { return "Host [name=" + name + ", user=" + user + ", threads=" + threads