DRILL-5811 reduced repeated log messages further. closes #965
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/5a100d13 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/5a100d13 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/5a100d13 Branch: refs/heads/master Commit: 5a100d1327a06a4509021916f302293ec228bbb3 Parents: 7bcc178 Author: Timothy Farkas <timothyfar...@apache.org> Authored: Thu Sep 28 12:56:18 2017 -0700 Committer: Paul Rogers <prog...@maprtech.com> Committed: Sat Sep 30 19:13:02 2017 -0700 ---------------------------------------------------------------------- .../exec/store/schedule/BlockMapBuilder.java | 24 +++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/5a100d13/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java index d898566..6b2e247 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/schedule/BlockMapBuilder.java @@ -104,12 +104,16 @@ public class BlockMapBuilder { @Override protected List<CompleteFileWork> runInner() throws Exception { final List<CompleteFileWork> work = Lists.newArrayList(); + + final Set<String> noDrillbitHosts = logger.isDebugEnabled() ? Sets.<String>newHashSet() : null; + boolean error = false; if (blockify && !compressed(status)) { try { ImmutableRangeMap<Long, BlockLocation> rangeMap = getBlockMap(status); for (Entry<Range<Long>, BlockLocation> l : rangeMap.asMapOfRanges().entrySet()) { - work.add(new CompleteFileWork(getEndpointByteMap(new FileStatusWork(status)), l.getValue().getOffset(), l.getValue().getLength(), status.getPath().toString())); + work.add(new CompleteFileWork(getEndpointByteMap(noDrillbitHosts, new FileStatusWork(status)), l.getValue().getOffset(), l.getValue().getLength(), status.getPath() + .toString())); } } catch (IOException e) { logger.warn("failure while generating file work.", e); @@ -119,7 +123,7 @@ public class BlockMapBuilder { if (!blockify || error || compressed(status)) { - work.add(new CompleteFileWork(getEndpointByteMap(new FileStatusWork(status)), 0, status.getLen(), status.getPath().toString())); + work.add(new CompleteFileWork(getEndpointByteMap(noDrillbitHosts, new FileStatusWork(status)), 0, status.getLen(), status.getPath().toString())); } // This if-condition is specific for empty CSV file @@ -127,7 +131,13 @@ public class BlockMapBuilder { // And if this CSV file is empty, rangeMap would be empty also // Therefore, at the point before this if-condition, work would not be populated if(work.isEmpty()) { - work.add(new CompleteFileWork(getEndpointByteMap(new FileStatusWork(status)), 0, 0, status.getPath().toString())); + work.add(new CompleteFileWork(getEndpointByteMap(noDrillbitHosts, new FileStatusWork(status)), 0, 0, status.getPath().toString())); + } + + if (noDrillbitHosts != null) { + for (String host: noDrillbitHosts) { + logger.debug("Failure finding Drillbit running on host {}. Skipping affinity to that host.", host); + } } return work; @@ -217,7 +227,7 @@ public class BlockMapBuilder { * @param work the FileWork to calculate endpoint bytes for * @throws IOException */ - public EndpointByteMap getEndpointByteMap(FileWork work) throws IOException { + public EndpointByteMap getEndpointByteMap(Set<String> noDrillbitHosts, FileWork work) throws IOException { Stopwatch watch = Stopwatch.createStarted(); Path fileName = new Path(work.getPath()); @@ -230,7 +240,6 @@ public class BlockMapBuilder { // Find submap of ranges that intersect with the rowGroup ImmutableRangeMap<Long,BlockLocation> subRangeMap = blockMap.subRangeMap(rowGroupRange); - final Set<String> noDrillbitHosts = logger.isDebugEnabled() ? Sets.<String>newHashSet() : null; // Iterate through each block in this submap and get the host for the block location for (Map.Entry<Range<Long>,BlockLocation> block : subRangeMap.asMapOfRanges().entrySet()) { @@ -249,8 +258,8 @@ public class BlockMapBuilder { DrillbitEndpoint endpoint = getDrillBitEndpoint(host); if (endpoint != null) { endpointByteMap.add(endpoint, bytes); - } else if (noDrillbitHosts != null && noDrillbitHosts.add(host)) { - logger.debug("Failure finding Drillbit running on host {}. Skipping affinity to that host.", host); + } else if (noDrillbitHosts != null) { + noDrillbitHosts.add(host); } } } @@ -279,5 +288,4 @@ public class BlockMapBuilder { logger.debug("Took {} ms to build endpoint map", watch.elapsed(TimeUnit.MILLISECONDS)); return ImmutableMap.copyOf(endpointMap); } - }
