Author: tgraves
Date: Wed Sep 26 15:22:21 2012
New Revision: 1390557
URL: http://svn.apache.org/viewvc?rev=1390557&view=rev
Log:
MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory in it.
(Robert Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
Wed Sep 26 15:22:21 2012
@@ -106,4 +106,20 @@ public interface LocalResource {
* localized
*/
public void setVisibility(LocalResourceVisibility visibility);
+
+ /**
+ * Get the <em>pattern</em> that should be used to extract entries from the
+ * archive (only used when type is <code>PATTERN</code>).
+ * @return <em>pattern</em> that should be used to extract entries from the
+ * archive.
+ */
+ public String getPattern();
+
+ /**
+ * Set the <em>pattern</em> that should be used to extract entries from the
+ * archive (only used when type is <code>PATTERN</code>).
+ * @param pattern <em>pattern</em> that should be used to extract entries
+ * from the archive.
+ */
+ public void setPattern(String pattern);
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
Wed Sep 26 15:22:21 2012
@@ -55,5 +55,14 @@ public enum LocalResourceType {
/**
* Regular file i.e. uninterpreted bytes.
*/
- FILE
+ FILE,
+
+ /**
+ * A hybrid between archive and file. Only part of the file is unarchived,
+ * and the original file is left in place, but in the same directory as the
+ * unarchived part. The part that is unarchived is determined by pattern
+ * in #{@link LocalResource}. Currently only jars support pattern, all
+ * others will be treated like a #{@link LocalResourceType#ARCHIVE}.
+ */
+ PATTERN
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
Wed Sep 26 15:22:21 2012
@@ -151,6 +151,25 @@ public class LocalResourcePBImpl extends
}
builder.setVisibility(convertToProtoFormat(visibility));
}
+
+ @Override
+ public synchronized String getPattern() {
+ LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasPattern()) {
+ return null;
+ }
+ return p.getPattern();
+ }
+
+ @Override
+ public synchronized void setPattern(String pattern) {
+ maybeInitBuilder();
+ if (pattern == null) {
+ builder.clearPattern();
+ return;
+ }
+ builder.setPattern(pattern);
+ }
private LocalResourceTypeProto convertToProtoFormat(LocalResourceType e) {
return ProtoUtils.convertToProtoFormat(e);
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
Wed Sep 26 15:22:21 2012
@@ -135,6 +135,7 @@ enum LocalResourceVisibilityProto {
enum LocalResourceTypeProto {
ARCHIVE = 1;
FILE = 2;
+ PATTERN = 3;
}
message LocalResourceProto {
@@ -142,7 +143,8 @@ message LocalResourceProto {
optional int64 size = 2;
optional int64 timestamp = 3;
optional LocalResourceTypeProto type = 4;
- optional LocalResourceVisibilityProto visibility= 5;
+ optional LocalResourceVisibilityProto visibility = 5;
+ optional string pattern = 6;
}
message ApplicationResourceUsageReportProto {
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
Wed Sep 26 15:22:21 2012
@@ -25,6 +25,7 @@ import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.Random;
import java.util.concurrent.Callable;
+import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -103,9 +104,9 @@ public class FSDownload implements Calla
return dCopy;
}
- private long unpack(File localrsrc, File dst) throws IOException {
+ private long unpack(File localrsrc, File dst, Pattern pattern) throws
IOException {
switch (resource.getType()) {
- case ARCHIVE:
+ case ARCHIVE: {
String lowerDst = dst.getName().toLowerCase();
if (lowerDst.endsWith(".jar")) {
RunJar.unJar(localrsrc, dst);
@@ -122,7 +123,39 @@ public class FSDownload implements Calla
+ "] to [" + dst + "]");
}
}
- break;
+ }
+ break;
+ case PATTERN: {
+ String lowerDst = dst.getName().toLowerCase();
+ if (lowerDst.endsWith(".jar")) {
+ RunJar.unJar(localrsrc, dst, pattern);
+ File newDst = new File(dst, dst.getName());
+ if (!dst.exists() && !dst.mkdir()) {
+ throw new IOException("Unable to create directory: [" + dst + "]");
+ }
+ if (!localrsrc.renameTo(newDst)) {
+ throw new IOException("Unable to rename file: [" + localrsrc
+ + "] to [" + newDst + "]");
+ }
+ } else if (lowerDst.endsWith(".zip")) {
+ LOG.warn("Treating [" + localrsrc + "] as an archive even though it " +
+ "was specified as PATTERN");
+ FileUtil.unZip(localrsrc, dst);
+ } else if (lowerDst.endsWith(".tar.gz") ||
+ lowerDst.endsWith(".tgz") ||
+ lowerDst.endsWith(".tar")) {
+ LOG.warn("Treating [" + localrsrc + "] as an archive even though it " +
+ "was specified as PATTERN");
+ FileUtil.unTar(localrsrc, dst);
+ } else {
+ LOG.warn("Cannot unpack " + localrsrc);
+ if (!localrsrc.renameTo(dst)) {
+ throw new IOException("Unable to rename file: [" + localrsrc
+ + "] to [" + dst + "]");
+ }
+ }
+ }
+ break;
case FILE:
default:
if (!localrsrc.renameTo(dst)) {
@@ -163,8 +196,13 @@ public class FSDownload implements Calla
public Path run() throws Exception {
return files.makeQualified(copy(sCopy, dst_work));
};
- });
- unpack(new File(dTmp.toUri()), new File(dFinal.toUri()));
+ });
+ Pattern pattern = null;
+ String p = resource.getPattern();
+ if(p != null) {
+ pattern = Pattern.compile(p);
+ }
+ unpack(new File(dTmp.toUri()), new File(dFinal.toUri()), pattern);
changePermissions(dFinal.getFileSystem(conf), dFinal);
files.rename(dst_work, destDirPath, Rename.OVERWRITE);
} catch (Exception e) {
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
Wed Sep 26 15:22:21 2012
@@ -107,8 +107,9 @@ public class TestFSDownload {
FileStatus status = files.getFileStatus(p);
ret.setSize(status.getLen());
ret.setTimestamp(status.getModificationTime());
- ret.setType(LocalResourceType.ARCHIVE);
+ ret.setType(LocalResourceType.PATTERN);
ret.setVisibility(vis);
+ ret.setPattern("classes/.*");
return ret;
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
Wed Sep 26 15:22:21 2012
@@ -209,6 +209,7 @@ public class ContainerLocalizer {
}
switch (rsrc.getType()) {
case ARCHIVE:
+ case PATTERN:
return 5 * rsrc.getSize();
case FILE:
default:
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
Wed Sep 26 15:22:21 2012
@@ -34,6 +34,7 @@ public class LocalResourceRequest
private final long timestamp;
private final LocalResourceType type;
private final LocalResourceVisibility visibility;
+ private final String pattern;
/**
* Wrap API resource to match against cache of localized resources.
@@ -45,22 +46,28 @@ public class LocalResourceRequest
this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
resource.getTimestamp(),
resource.getType(),
- resource.getVisibility());
+ resource.getVisibility(),
+ resource.getPattern());
}
LocalResourceRequest(Path loc, long timestamp, LocalResourceType type,
- LocalResourceVisibility visibility) {
+ LocalResourceVisibility visibility, String pattern) {
this.loc = loc;
this.timestamp = timestamp;
this.type = type;
this.visibility = visibility;
+ this.pattern = pattern;
}
@Override
public int hashCode() {
- return loc.hashCode() ^
+ int hash = loc.hashCode() ^
(int)((timestamp >>> 32) ^ timestamp) *
type.hashCode();
+ if(pattern != null) {
+ hash = hash ^ pattern.hashCode();
+ }
+ return hash;
}
@Override
@@ -72,9 +79,14 @@ public class LocalResourceRequest
return false;
}
final LocalResourceRequest other = (LocalResourceRequest) o;
+ String pattern = getPattern();
+ String otherPattern = other.getPattern();
+ boolean patternEquals = (pattern == null && otherPattern == null) ||
+ (pattern != null && otherPattern != null &&
pattern.equals(otherPattern));
return getPath().equals(other.getPath()) &&
getTimestamp() == other.getTimestamp() &&
- getType() == other.getType();
+ getType() == other.getType() &&
+ patternEquals;
}
@Override
@@ -87,6 +99,19 @@ public class LocalResourceRequest
ret = (int)(getTimestamp() - other.getTimestamp());
if (0 == ret) {
ret = getType().ordinal() - other.getType().ordinal();
+ if (0 == ret) {
+ String pattern = getPattern();
+ String otherPattern = other.getPattern();
+ if (pattern == null && otherPattern == null) {
+ ret = 0;
+ } else if (pattern == null) {
+ ret = -1;
+ } else if (otherPattern == null) {
+ ret = 1;
+ } else {
+ ret = pattern.compareTo(otherPattern);
+ }
+ }
}
}
return ret;
@@ -122,6 +147,11 @@ public class LocalResourceRequest
}
@Override
+ public String getPattern() {
+ return pattern;
+ }
+
+ @Override
public void setResource(URL resource) {
throw new UnsupportedOperationException();
}
@@ -145,14 +175,20 @@ public class LocalResourceRequest
public void setVisibility(LocalResourceVisibility visibility) {
throw new UnsupportedOperationException();
}
-
+
+ @Override
+ public void setPattern(String pattern) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{ ");
sb.append(getPath().toString()).append(", ");
sb.append(getTimestamp()).append(", ");
- sb.append(getType()).append(" }");
+ sb.append(getType()).append(", ");
+ sb.append(getPattern()).append(" }");
return sb.toString();
}
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
Wed Sep 26 15:22:21 2012
@@ -219,7 +219,8 @@ public class LocalizedResource implement
ContainerId container = ctxt.getContainerId();
rsrc.ref.add(container);
rsrc.dispatcher.getEventHandler().handle(
- new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt));
+ new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt,
+ req.getLocalResourceRequest().getPattern()));
}
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
Wed Sep 26 15:22:21 2012
@@ -752,6 +752,7 @@ public class ResourceLocalizationService
next.setTimestamp(nextRsrc.getTimestamp());
next.setType(nextRsrc.getType());
next.setVisibility(evt.getVisibility());
+ next.setPattern(evt.getPattern());
scheduled.put(nextRsrc, evt);
return next;
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
Wed Sep 26 15:22:21 2012
@@ -32,14 +32,16 @@ public class LocalizerResourceRequestEve
private final LocalizerContext context;
private final LocalizedResource resource;
private final LocalResourceVisibility vis;
+ private final String pattern;
public LocalizerResourceRequestEvent(LocalizedResource resource,
- LocalResourceVisibility vis, LocalizerContext context) {
+ LocalResourceVisibility vis, LocalizerContext context, String pattern) {
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
ConverterUtils.toString(context.getContainerId()));
this.vis = vis;
this.context = context;
this.resource = resource;
+ this.pattern = pattern;
}
public LocalizedResource getResource() {
@@ -54,4 +56,8 @@ public class LocalizerResourceRequestEve
return vis;
}
+ public String getPattern() {
+ return pattern;
+ }
+
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
Wed Sep 26 15:22:21 2012
@@ -37,7 +37,7 @@ import static org.junit.Assert.*;
public class TestLocalResource {
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path
p, long size,
- long timestamp, LocalResourceType type, LocalResourceVisibility state)
+ long timestamp, LocalResourceType type, LocalResourceVisibility state,
String pattern)
throws URISyntaxException {
org.apache.hadoop.yarn.api.records.LocalResource ret =
RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
@@ -45,6 +45,7 @@ public class TestLocalResource {
ret.setTimestamp(timestamp);
ret.setType(type);
ret.setVisibility(state);
+ ret.setPattern(pattern);
return ret;
}
@@ -72,9 +73,9 @@ public class TestLocalResource {
long basetime = r.nextLong() >>> 2;
org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
- new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PUBLIC,
null);
org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
- new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PUBLIC,
null);
final LocalResourceRequest a = new LocalResourceRequest(yA);
LocalResourceRequest b = new LocalResourceRequest(yA);
checkEqual(a, b);
@@ -83,31 +84,37 @@ public class TestLocalResource {
// ignore visibility
yB = getYarnResource(
- new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PRIVATE);
+ new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PRIVATE,
null);
b = new LocalResourceRequest(yB);
checkEqual(a, b);
// ignore size
yB = getYarnResource(
- new Path("http://yak.org:80/foobar";), 0, basetime, FILE, PRIVATE);
+ new Path("http://yak.org:80/foobar";), 0, basetime, FILE, PRIVATE,
null);
b = new LocalResourceRequest(yB);
checkEqual(a, b);
// note path
yB = getYarnResource(
- new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC);
+ new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC,
null);
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
// note type
yB = getYarnResource(
- new Path("http://yak.org:80/foobar";), 0, basetime, ARCHIVE, PUBLIC);
+ new Path("http://yak.org:80/foobar";), 0, basetime, ARCHIVE, PUBLIC,
null);
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
// note timestamp
yB = getYarnResource(
- new Path("http://yak.org:80/foobar";), 0, basetime + 1, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar";), 0, basetime + 1, FILE, PUBLIC,
null);
+ b = new LocalResourceRequest(yB);
+ checkNotEqual(a, b);
+
+ // note pattern
+ yB = getYarnResource(
+ new Path("http://yak.org:80/foobar";), 0, basetime + 1, FILE, PUBLIC,
"^/foo/.*");
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
}
@@ -120,24 +127,35 @@ public class TestLocalResource {
System.out.println("SEED: " + seed);
long basetime = r.nextLong() >>> 2;
org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
- new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar";), -1, basetime, FILE, PUBLIC,
"^/foo/.*");
final LocalResourceRequest a = new LocalResourceRequest(yA);
// Path primary
org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
- new Path("http://yak.org:80/foobaz";), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobaz";), -1, basetime, FILE, PUBLIC,
"^/foo/.*");
LocalResourceRequest b = new LocalResourceRequest(yB);
assertTrue(0 > a.compareTo(b));
// timestamp secondary
yB = getYarnResource(
- new Path("http://yak.org:80/foobar";), -1, basetime + 1, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar";), -1, basetime + 1, FILE, PUBLIC,
"^/foo/.*");
b = new LocalResourceRequest(yB);
assertTrue(0 > a.compareTo(b));
// type tertiary
yB = getYarnResource(
- new Path("http://yak.org:80/foobar";), -1, basetime, ARCHIVE, PUBLIC);
+ new Path("http://yak.org:80/foobar";), -1, basetime, ARCHIVE, PUBLIC,
"^/foo/.*");
+ b = new LocalResourceRequest(yB);
+ assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
+
+ // path 4th
+ yB = getYarnResource(
+ new Path("http://yak.org:80/foobar";), -1, basetime, ARCHIVE, PUBLIC,
"^/food/.*");
+ b = new LocalResourceRequest(yB);
+ assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
+
+ yB = getYarnResource(
+ new Path("http://yak.org:80/foobar";), -1, basetime, ARCHIVE, PUBLIC,
null);
b = new LocalResourceRequest(yB);
assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
Wed Sep 26 15:22:21 2012
@@ -230,7 +230,7 @@ public class TestLocalResourcesTrackerIm
long ts, LocalResourceVisibility vis) {
final LocalResourceRequest req =
new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i),
- ts + i * 2000, LocalResourceType.FILE, vis);
+ ts + i * 2000, LocalResourceType.FILE, vis, null);
return req;
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java?rev=1390557&r1=1390556&r2=1390557&view=diff
==============================================================================
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
Wed Sep 26 15:22:21 2012
@@ -83,7 +83,7 @@ public class TestResourceRetention {
for (int i = 0; i < nRsrcs; ++i) {
final LocalResourceRequest req = new LocalResourceRequest(
new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
- LocalResourceType.FILE, LocalResourceVisibility.PUBLIC);
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
final long ts = timestamp + i * tsstep;
final Path p = new Path("file:///local/" + user + "/rsrc" + i);
LocalizedResource rsrc = new LocalizedResource(req, null) {
