TAJO-1081: Non-forwarded (simple) query shows wrong rows. Closes #175
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3d029fa9 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3d029fa9 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3d029fa9 Branch: refs/heads/block_iteration Commit: 3d029fa90e483e6d0678b8b6c72c4a2410e97399 Parents: d1022fb Author: Hyunsik Choi <[email protected]> Authored: Thu Oct 2 00:27:24 2014 -0700 Committer: Hyunsik Choi <[email protected]> Committed: Thu Oct 2 00:27:24 2014 -0700 ---------------------------------------------------------------------- CHANGES | 2 ++ .../tajo/cli/DefaultTajoCliOutputFormatter.java | 33 ++++++++++++++------ .../org/apache/tajo/cli/DescTableCommand.java | 6 +++- .../java/org/apache/tajo/client/TajoClient.java | 4 ++- .../apache/tajo/engine/planner/PlannerUtil.java | 3 +- .../org/apache/tajo/master/GlobalEngine.java | 6 ++++ .../tajo/storage/AbstractStorageManager.java | 2 +- 7 files changed, 43 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index c73490c..3876a1e 100644 --- a/CHANGES +++ b/CHANGES @@ -155,6 +155,8 @@ Release 0.9.0 - unreleased BUG FIXES + TAJO-1081: Non-forwarded (simple) query shows wrong rows. (hyunsik) + TAJO-981: Help command (\?) in tsql takes too long time. (YeonSu Han via jaehwa) TAJO-962: Column reference used in LIMIT clause incurs NPE. http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java index 543ac8c..b36f521 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java @@ -24,6 +24,7 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.client.QueryStatus; +import org.apache.tajo.client.TajoClient; import org.apache.tajo.util.FileUtil; import java.io.InputStream; @@ -50,13 +51,25 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { this.printPause = false; } - private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows, String postfix) { + private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows, String postfix, + boolean endOfTuple) { TableStats stat = tableDesc.getStats(); - String volume = stat == null ? "0 B" : FileUtil.humanReadableByteCount(stat.getNumBytes(), false); - long resultRows = stat == null ? 0 : stat.getNumRows(); + String volume = stat == null ? "unknown bytes" : FileUtil.humanReadableByteCount(stat.getNumBytes(), false); + long resultRows = stat == null ? TajoClient.UNKNOWN_ROW_NUMBER : stat.getNumRows(); - long realNumRows = resultRows != 0 ? resultRows : totalPrintedRows; - return "(" + realNumRows + " rows, " + getResponseTimeReadable(responseTime) + ", " + volume + " " + postfix + ")"; + String displayRowNum; + if (resultRows == TajoClient.UNKNOWN_ROW_NUMBER) { + + if (endOfTuple) { + displayRowNum = totalPrintedRows + " rows"; + } else { + displayRowNum = "unknown row number"; + } + + } else { + displayRowNum = resultRows + " rows"; + } + return "(" + displayRowNum + ", " + getResponseTimeReadable(responseTime) + ", " + volume + " " + postfix + ")"; } protected String getResponseTimeReadable(float responseTime) { @@ -66,13 +79,13 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { @Override public void printResult(PrintWriter sout, InputStream sin, TableDesc tableDesc, float responseTime, ResultSet res) throws Exception { - long resultRows = tableDesc.getStats() == null ? 0 : tableDesc.getStats().getNumRows(); - if (resultRows == 0) { + long resultRows = tableDesc.getStats() == null ? -1 : tableDesc.getStats().getNumRows(); + if (resultRows == -1) { resultRows = Integer.MAX_VALUE; } if (res == null) { - sout.println(getQuerySuccessMessage(tableDesc, responseTime, 0, "inserted")); + sout.println(getQuerySuccessMessage(tableDesc, responseTime, 0, "inserted", true)); return; } ResultSetMetaData rsmd = res.getMetaData(); @@ -86,6 +99,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { int numOfPrintedRows = 0; int totalPrintedRows = 0; + boolean endOfTuple = true; while (res.next()) { for (int i = 1; i <= numOfColumns; i++) { if (i > 1) sout.print(", "); @@ -109,6 +123,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { sout.flush(); if (sin != null) { if (sin.read() == 'q') { + endOfTuple = false; sout.println(); break; } @@ -117,7 +132,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter { sout.println(); } } - sout.println(getQuerySuccessMessage(tableDesc, responseTime, totalPrintedRows, "selected")); + sout.println(getQuerySuccessMessage(tableDesc, responseTime, totalPrintedRows, "selected", endOfTuple)); sout.flush(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java index d8023f2..8fab138 100644 --- a/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java +++ b/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java @@ -23,6 +23,7 @@ import org.apache.commons.lang.StringEscapeUtils; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; +import org.apache.tajo.client.TajoClient; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.TUtil; @@ -79,7 +80,10 @@ public class DescTableCommand extends TajoShellCommand { sb.append("table path: ").append(desc.getPath()).append("\n"); sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n"); if (desc.getStats() != null) { - sb.append("number of rows: ").append(desc.getStats().getNumRows()).append("\n"); + + long row = desc.getStats().getNumRows(); + String rowText = row == TajoClient.UNKNOWN_ROW_NUMBER ? "unknown" : row + ""; + sb.append("number of rows: ").append(rowText).append("\n"); sb.append("volume: ").append( FileUtil.humanReadableByteCount(desc.getStats().getNumBytes(), true)).append("\n"); http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java ---------------------------------------------------------------------- diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java index 3d61cce..2288851 100644 --- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java +++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java @@ -73,6 +73,8 @@ import static org.apache.tajo.ipc.ClientProtos.SerializedResultSet; @ThreadSafe public class TajoClient implements Closeable { + public static final int UNKNOWN_ROW_NUMBER = -1; + private final Log LOG = LogFactory.getLog(TajoClient.class); private final TajoConf conf; @@ -413,7 +415,7 @@ public class TajoClient implements Closeable { } } else { // If a non-forwarded insert into query - if (queryId.equals(QueryIdFactory.NULL_QUERY_ID) && response.getMaxRowNum() < 0) { + if (queryId.equals(QueryIdFactory.NULL_QUERY_ID) && response.getMaxRowNum() == 0) { return this.createNullResultSet(queryId); } else { if (response.hasResultSet() || response.hasTableDesc()) { http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java index c4cc254..827be83 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java @@ -38,6 +38,7 @@ import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.exception.InvalidQueryException; import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -854,7 +855,7 @@ public class PlannerUtil { int startFileIndex, int numResultFiles, AtomicInteger currentFileIndex) throws IOException { if (fs.isDirectory(path)) { - FileStatus[] files = fs.listStatus(path); + FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter); if (files != null && files.length > 0) { for (FileStatus eachFile : files) { if (result.size() >= numResultFiles) { http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 9612eeb..1537c89 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -39,6 +39,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.statistics.TableStats; +import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; @@ -750,6 +751,11 @@ public class GlobalEngine extends AbstractService { TableStats stats = new TableStats(); stats.setNumBytes(totalSize); + + if (isExternal) { // if it is an external table, there is no way to know the exact row number without processing. + stats.setNumRows(TajoClient.UNKNOWN_ROW_NUMBER); + } + TableDesc desc = new TableDesc(CatalogUtil.buildFQName(databaseName, simpleTableName), schema, meta, path, isExternal); desc.setStats(stats); http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java index c6be73b..4bf0057 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java @@ -304,7 +304,7 @@ public abstract class AbstractStorageManager { // FileInputFormat Area ///////////////////////////////////////////////////////////////////////////// - private static final PathFilter hiddenFileFilter = new PathFilter() { + public static final PathFilter hiddenFileFilter = new PathFilter() { public boolean accept(Path p) { String name = p.getName(); return !name.startsWith("_") && !name.startsWith(".");
