Author: cws Date: Sun Sep 9 22:23:53 2012 New Revision: 1382600 URL: http://svn.apache.org/viewvc?rev=1382600&view=rev Log: HIVE-3387. Meta data file size exceeds limit (Navis Ryu via cws)
Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1382600&r1=1382599&r2=1382600&view=diff ============================================================================== --- hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original) +++ hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Sun Sep 9 22:23:53 2012 @@ -28,7 +28,10 @@ import java.net.URL; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import javax.security.auth.Subject; import javax.security.auth.login.LoginException; @@ -160,10 +163,17 @@ public class Hadoop20Shims implements Ha } public InputSplitShim(CombineFileSplit old) throws IOException { - super(old); + super(old.getJob(), old.getPaths(), old.getStartOffsets(), + old.getLengths(), dedup(old.getLocations())); _isShrinked = false; } + private static String[] dedup(String[] locations) { + Set<String> dedup = new HashSet<String>(); + Collections.addAll(dedup, locations); + return dedup.toArray(new String[dedup.size()]); + } + @Override public void shrinkSplit(long length) { _isShrinked = true; Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1382600&r1=1382599&r2=1382600&view=diff ============================================================================== --- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original) +++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Sun Sep 9 22:23:53 2012 @@ -25,7 +25,10 @@ import java.net.URI; import java.net.URISyntaxException; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -165,10 +168,17 @@ public abstract class HadoopShimsSecure } public InputSplitShim(CombineFileSplit old) throws IOException { - super(old); + super(old.getJob(), old.getPaths(), old.getStartOffsets(), + old.getLengths(), dedup(old.getLocations())); _isShrinked = false; } + private static String[] dedup(String[] locations) { + Set<String> dedup = new HashSet<String>(); + Collections.addAll(dedup, locations); + return dedup.toArray(new String[dedup.size()]); + } + @Override public void shrinkSplit(long length) { _isShrinked = true;