Author: eli Date: Sat Oct 27 04:30:41 2012 New Revision: 1402741 URL: http://svn.apache.org/viewvc?rev=1402741&view=rev Log: HADOOP-8968. Add a flag to completely disable the worker version check. Contributed by Alejandro Abdelnur
Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/core/core-default.xml hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1402741&r1=1402740&r2=1402741&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Sat Oct 27 04:30:41 2012 @@ -107,6 +107,9 @@ Release 1.2.0 - unreleased HADOOP-8567. Port conf servlet to dump running configuration to branch 1.x. (Jing Zhao via suresh) + HADOOP-8968. Add a flag to completely disable the worker version check. + (tucu via eli) + OPTIMIZATIONS HDFS-2533. Backport: Remove needless synchronization on some FSDataSet Modified: hadoop/common/branches/branch-1/src/core/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/core-default.xml?rev=1402741&r1=1402740&r2=1402741&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/core/core-default.xml (original) +++ hadoop/common/branches/branch-1/src/core/core-default.xml Sat Oct 27 04:30:41 2012 @@ -578,6 +578,20 @@ </property> <property> + <name>hadoop.skip.worker.version.check</name> + <value>false</value> + <description> + By default datanodes refuse to connect to namenodes if their build + revision (svn revision) do not match, and tasktrackers refuse to + connect to jobtrackers if their build version (version, revision, + user, and source checksum) do not match. This option changes the + behavior of hadoop workers to skip doing a version check at all. + This option supersedes the 'hadoop.relaxed.worker.version.check' + option. + </description> +</property> + +<property> <name>hadoop.jetty.logs.serve.aliases</name> <value>true</value> <description> Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=1402741&r1=1402740&r2=1402741&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java (original) +++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/fs/CommonConfigurationKeys.java Sat Oct 27 04:30:41 2012 @@ -74,6 +74,11 @@ public class CommonConfigurationKeys { "hadoop.relaxed.worker.version.check"; public static final boolean HADOOP_RELAXED_VERSION_CHECK_DEFAULT = false; + /** See src/core/core-default.xml */ + public static final String HADOOP_SKIP_VERSION_CHECK_KEY = + "hadoop.skip.worker.version.check"; + public static final boolean HADOOP_SKIP_VERSION_CHECK_DEFAULT = false; + /** Enable/Disable aliases serving from jetty */ public static final String HADOOP_JETTY_LOGS_SERVE_ALIASES = "hadoop.jetty.logs.serve.aliases"; Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1402741&r1=1402740&r2=1402741&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original) +++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNode.java Sat Oct 27 04:30:41 2012 @@ -77,11 +77,9 @@ import org.apache.hadoop.hdfs.security.t import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager; import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager.AccessMode; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; -import org.apache.hadoop.hdfs.server.common.GenerationStamp; import org.apache.hadoop.hdfs.server.common.HdfsConstants; import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.IncorrectVersionException; -import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.FSDataset.VolumeInfo; import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources; import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeInstrumentation; @@ -244,6 +242,11 @@ public class DataNode extends Configured private boolean relaxedVersionCheck; /** + * Whether the DN completely skips version check with the NN. + */ + private boolean noVersionCheck; + + /** * Testing hook that allows tests to delay the sending of blockReceived RPCs * to the namenode. This can help find bugs in append. */ @@ -364,6 +367,9 @@ public class DataNode extends Configured this.relaxedVersionCheck = conf.getBoolean( CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY, CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_DEFAULT); + noVersionCheck = conf.getBoolean( + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY, + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_DEFAULT); InetSocketAddress socAddr = DataNode.getStreamingAddr(conf); int tmpPort = socAddr.getPort(); @@ -600,22 +606,31 @@ public class DataNode extends Configured nsInfo.getVersion().equals(VersionInfo.getVersion()); boolean revisionMatch = nsInfo.getRevision().equals(VersionInfo.getRevision()); - if (revisionMatch && !versionMatch) { throw new AssertionError("Invalid build. The revisions match" + " but the NN version is " + nsInfo.getVersion() + " and the DN version is " + VersionInfo.getVersion()); } - if (relaxedVersionCheck) { - if (versionMatch && !revisionMatch) { - LOG.info("Permitting datanode revision " + VersionInfo.getRevision() + - " to connect to namenode revision " + nsInfo.getRevision() + - " because " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY + - " is enabled"); - } - return versionMatch; + if (noVersionCheck) { + LOG.info("Permitting datanode version '" + VersionInfo.getVersion() + + "' and revision '" + VersionInfo.getRevision() + + "' to connect to namenode version '" + nsInfo.getVersion() + + "' and revision '" + nsInfo.getRevision() + "' because " + + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY + + " is enabled"); + return true; } else { - return revisionMatch; + if (relaxedVersionCheck) { + if (versionMatch && !revisionMatch) { + LOG.info("Permitting datanode revision " + VersionInfo.getRevision() + + " to connect to namenode revision " + nsInfo.getRevision() + + " because " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY + + " is enabled"); + } + return versionMatch; + } else { + return revisionMatch; + } } } @@ -633,12 +648,15 @@ public class DataNode extends Configured } } if (!isPermittedVersion(nsInfo)) { - String errorMsg = "Incompatible versions: namenode version " + - nsInfo.getVersion() + " revision " + nsInfo.getRevision() + - " datanode version " + VersionInfo.getVersion() + " revision " + - VersionInfo.getRevision() + " and " + - CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY + - " is " + (relaxedVersionCheck ? "enabled" : "not enabled"); + String errorMsg = "Shutting down. Incompatible version or revision." + + "DataNode version '" + VersionInfo.getVersion() + + "' and revision '" + VersionInfo.getRevision() + + "' and NameNode version '" + nsInfo.getVersion() + + "' and revision '" + nsInfo.getRevision() + + " and " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY + + " is " + (relaxedVersionCheck ? "enabled" : "not enabled") + + " and " + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY + + " is " + (noVersionCheck ? "enabled" : "not enabled"); LOG.fatal(errorMsg); notifyNamenode(DatanodeProtocol.NOTIFY, errorMsg); throw new IOException( errorMsg ); Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java?rev=1402741&r1=1402740&r2=1402741&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskTracker.java Sat Oct 27 04:30:41 2012 @@ -418,6 +418,11 @@ public class TaskTracker implements MRCo */ private boolean relaxedVersionCheck; + /** + * Whether the TT completely skips version check with the JT. + */ + private boolean skipVersionCheck; + /* * A list of commitTaskActions for whom commit response has been received */ @@ -1459,6 +1464,9 @@ public class TaskTracker implements MRCo relaxedVersionCheck = conf.getBoolean( CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY, CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_DEFAULT); + skipVersionCheck = conf.getBoolean( + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY, + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_DEFAULT); FILE_CACHE_SIZE = conf.getInt("mapred.tasktracker.file.cache.size", 2000); maxMapSlots = conf.getInt( "mapred.tasktracker.map.tasks.maximum", 2); @@ -1623,7 +1631,7 @@ public class TaskTracker implements MRCo private long getHeartbeatInterval(int numFinishedTasks) { return (heartbeatInterval / (numFinishedTasks * oobHeartbeatDamper + 1)); } - + /** * @return true if this tasktracker is permitted to connect to * the given jobtracker version @@ -1637,16 +1645,26 @@ public class TaskTracker implements MRCo " but the JT version is " + jtVersion + " and the TT version is " + VersionInfo.getVersion()); } - if (relaxedVersionCheck) { - if (!buildVersionMatch && versionMatch) { - LOG.info("Permitting tasktracker revision " + VersionInfo.getRevision() + - " to connect to jobtracker " + jtBuildVersion + " because " + - CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY + - " is enabled"); - } - return versionMatch; + if (skipVersionCheck) { + LOG.info("Permitting tasktracker version '" + VersionInfo.getVersion() + + "' and build '" + VersionInfo.getBuildVersion() + + "' to connect to jobtracker version '" + jtVersion + + "' and build '" + jtBuildVersion + "' because " + + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY + + " is enabled"); + return true; } else { - return buildVersionMatch; + if (relaxedVersionCheck) { + if (!buildVersionMatch && versionMatch) { + LOG.info("Permitting tasktracker build " + VersionInfo.getBuildVersion() + + " to connect to jobtracker build " + jtBuildVersion + " because " + + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY + + " is enabled"); + } + return versionMatch; + } else { + return buildVersionMatch; + } } } @@ -1689,11 +1707,15 @@ public class TaskTracker implements MRCo String jtBuildVersion = jobClient.getBuildVersion(); String jtVersion = jobClient.getVIVersion(); if (!isPermittedVersion(jtBuildVersion, jtVersion)) { - String msg = "Shutting down. Incompatible buildVersion." + - "\nJobTracker's: " + jtBuildVersion + - "\nTaskTracker's: "+ VersionInfo.getBuildVersion() + + String msg = "Shutting down. Incompatible version or build version." + + "TaskTracker version '" + VersionInfo.getVersion() + + "' and build '" + VersionInfo.getBuildVersion() + + "' and JobTracker version '" + jtVersion + + "' and build '" + jtBuildVersion + " and " + CommonConfigurationKeys.HADOOP_RELAXED_VERSION_CHECK_KEY + - " is " + (relaxedVersionCheck ? "enabled" : "not enabled"); + " is " + (relaxedVersionCheck ? "enabled" : "not enabled") + + " and " + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY + + " is " + (skipVersionCheck ? "enabled" : "not enabled"); LOG.fatal(msg); try { jobClient.reportTaskTrackerError(taskTrackerName, null, msg); Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java?rev=1402741&r1=1402740&r2=1402741&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVersionCheck.java Sat Oct 27 04:30:41 2012 @@ -116,4 +116,42 @@ public class TestDataNodeVersionCheck { } } } + + /** + * Test no DN version checking + */ + @Test + public void testNoVersionCheck() throws IOException { + MiniDFSCluster cluster = null; + try { + Configuration conf = new Configuration(); + conf.setBoolean( + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY, true); + cluster = new MiniDFSCluster(conf, 1, true, null); + + DataNode dn = cluster.getDataNodes().get(0); + + final NamespaceInfo currInfo = new NamespaceInfo(0, 0, 0); + assertTrue(dn.isPermittedVersion(currInfo)); + + // Different revisions are permitted + NamespaceInfo infoDiffRev = new NamespaceInfo(0, 0, 0) { + @Override public String getRevision() { return "bogus"; } + }; + assertTrue("Different revisions should be permitted", + dn.isPermittedVersion(infoDiffRev)); + + // Different versions are permitted + NamespaceInfo infoDiffVersion = new NamespaceInfo(0, 0, 0) { + @Override public String getVersion() { return "bogus"; } + @Override public String getRevision() { return "bogus"; } + }; + assertTrue("Different versions should be permitted", + dn.isPermittedVersion(infoDiffVersion)); + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } \ No newline at end of file Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java?rev=1402741&r1=1402740&r2=1402741&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestTaskTrackerVersionCheck.java Sat Oct 27 04:30:41 2012 @@ -88,4 +88,29 @@ public class TestTaskTrackerVersionCheck } } } + + /** + * Test no TT version checking + */ + @Test + public void testNoVersionCheck() throws IOException { + MiniMRCluster mr = null; + try { + JobConf jtConf = new JobConf(); + jtConf.setBoolean( + CommonConfigurationKeys.HADOOP_SKIP_VERSION_CHECK_KEY, true); + mr = new MiniMRCluster(1, "file:///", 1, null, null, jtConf); + TaskTracker tt = mr.getTaskTrackerRunner(0).getTaskTracker(); + String currFullVersion = VersionInfo.getBuildVersion(); + String currVersion = VersionInfo.getVersion(); + + assertTrue(tt.isPermittedVersion(currFullVersion, currVersion)); + assertTrue(tt.isPermittedVersion(currFullVersion+"x", currVersion+"x")); + assertTrue(tt.isPermittedVersion(currFullVersion+"x", currVersion)); + } finally { + if (mr != null) { + mr.shutdown(); + } + } + } }