Author: daijy Date: Mon May 17 01:30:06 2010 New Revision: 944949 URL: http://svn.apache.org/viewvc?rev=944949&view=rev Log: PIG-1403: Make Pig work with remote HDFS in secure mode
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=944949&r1=944948&r2=944949&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Mon May 17 01:30:06 2010 @@ -1384,35 +1384,37 @@ LogicalOperator LoadClause(LogicalPlan l if (absolutePath == null) { absolutePath = loFunc.relativeToAbsolutePath(filename, getCurrentDir(pigContext)); - // Get native host - String defaultFS = (String)pigContext.getProperties().get("fs.default.name"); - URI defaultFSURI = new URI(defaultFS); - String defaultHost = defaultFSURI.getHost(); - if (defaultHost==null) - defaultHost=""; - defaultHost = defaultHost.toLowerCase(); - - Set<String> remoteHosts = getRemoteHosts(absolutePath, defaultHost); - - String hdfsServersString = (String)pigContext.getProperties().get("mapreduce.job.hdfs-servers"); - if (hdfsServersString==null) hdfsServersString=""; - String hdfsServers[] = hdfsServersString.split(","); - - for (String remoteHost : remoteHosts) { - boolean existing = false; - for (String hdfsServer:hdfsServers) { - if (hdfsServer.equals(remoteHost)) - existing = true; - } - if (!existing) { - if (!hdfsServersString.isEmpty()) - hdfsServersString = hdfsServersString + ","; - hdfsServersString = hdfsServersString + remoteHost; - } - } - - if (!hdfsServersString.isEmpty()) - pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", hdfsServersString); + if (absolutePath!=null) { + // Get native host + String defaultFS = (String)pigContext.getProperties().get("fs.default.name"); + URI defaultFSURI = new URI(defaultFS); + String defaultHost = defaultFSURI.getHost(); + if (defaultHost==null) + defaultHost=""; + defaultHost = defaultHost.toLowerCase(); + + Set<String> remoteHosts = getRemoteHosts(absolutePath, defaultHost); + + String hdfsServersString = (String)pigContext.getProperties().get("mapreduce.job.hdfs-servers"); + if (hdfsServersString==null) hdfsServersString=""; + String hdfsServers[] = hdfsServersString.split(","); + + for (String remoteHost : remoteHosts) { + boolean existing = false; + for (String hdfsServer:hdfsServers) { + if (hdfsServer.equals(remoteHost)) + existing = true; + } + if (!existing) { + if (!hdfsServersString.isEmpty()) + hdfsServersString = hdfsServersString + ","; + hdfsServersString = hdfsServersString + remoteHost; + } + } + + if (!hdfsServersString.isEmpty()) + pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", hdfsServersString); + } fileNameMap.put(constructFileNameSignature(filename, funcSpec), absolutePath); } lo = new LOLoad(lp, new OperatorKey(scope, getNextId()), new FileSpec(absolutePath, funcSpec),