YARN-6654. RollingLevelDBTimelineStore backwards incompatible after fst upgrade. Contributed by Jonathan Eagles
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f1ee72b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f1ee72b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f1ee72b Branch: refs/heads/HDFS-7240 Commit: 5f1ee72b0ebf0330417b7c0115083bc851923be4 Parents: 945c095 Author: Jason Lowe <jl...@yahoo-inc.com> Authored: Thu Jul 13 17:27:40 2017 -0500 Committer: Jason Lowe <jl...@yahoo-inc.com> Committed: Thu Jul 13 17:27:40 2017 -0500 ---------------------------------------------------------------------- .../timeline/RollingLevelDBTimelineStore.java | 50 ++++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f1ee72b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java index d139346..00f6630 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/RollingLevelDBTimelineStore.java @@ -28,6 +28,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -74,6 +75,7 @@ import org.iq80.leveldb.Options; import org.iq80.leveldb.ReadOptions; import org.iq80.leveldb.WriteBatch; import org.nustaq.serialization.FSTConfiguration; +import org.nustaq.serialization.FSTClazzNameRegistry; import static java.nio.charset.StandardCharsets.UTF_8; @@ -170,9 +172,22 @@ public class RollingLevelDBTimelineStore extends AbstractService implements .getLog(RollingLevelDBTimelineStore.class); private static FSTConfiguration fstConf = FSTConfiguration.createDefaultConfiguration(); + // Fall back to 2.24 parsing if 2.50 parsing fails + private static FSTConfiguration fstConf224 = + FSTConfiguration.createDefaultConfiguration(); + // Static class code for 2.24 + private static final int LINKED_HASH_MAP_224_CODE = 83; static { fstConf.setShareReferences(false); + fstConf224.setShareReferences(false); + // YARN-6654 unable to find class for code 83 (LinkedHashMap) + // The linked hash map was changed between 2.24 and 2.50 so that + // the static code for LinkedHashMap (83) was changed to a dynamic + // code. + FSTClazzNameRegistry registry = fstConf224.getClassRegistry(); + registry.registerClass( + LinkedHashMap.class, LINKED_HASH_MAP_224_CODE, fstConf224); } @Private @@ -339,7 +354,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements deletionThread.start(); } super.serviceStart(); - } + } @Override protected void serviceStop() throws Exception { @@ -365,7 +380,7 @@ public class RollingLevelDBTimelineStore extends AbstractService implements private final long ttl; private final long ttlInterval; - public EntityDeletionThread(Configuration conf) { + EntityDeletionThread(Configuration conf) { ttl = conf.getLong(TIMELINE_SERVICE_TTL_MS, DEFAULT_TIMELINE_SERVICE_TTL_MS); ttlInterval = conf.getLong( @@ -479,9 +494,15 @@ public class RollingLevelDBTimelineStore extends AbstractService implements try { o = fstConf.asObject(iterator.peekNext().getValue()); entity.addOtherInfo(keyStr, o); - } catch (Exception e) { - LOG.warn("Error while decoding " - + entityId + ":otherInfo:" + keyStr, e); + } catch (Exception ignore) { + try { + // Fall back to 2.24 parser + o = fstConf224.asObject(iterator.peekNext().getValue()); + entity.addOtherInfo(keyStr, o); + } catch (Exception e) { + LOG.warn("Error while decoding " + + entityId + ":otherInfo:" + keyStr, e); + } } } } else if (key[prefixlen] == RELATED_ENTITIES_COLUMN[0]) { @@ -1348,8 +1369,13 @@ public class RollingLevelDBTimelineStore extends AbstractService implements Object o = null; try { o = fstConf.asObject(value); - } catch (Exception e) { - LOG.warn("Error while decoding " + tstype, e); + } catch (Exception ignore) { + try { + // Fall back to 2.24 parser + o = fstConf224.asObject(value); + } catch (Exception e) { + LOG.warn("Error while decoding " + tstype, e); + } } if (o == null) { event.setEventInfo(null); @@ -1378,8 +1404,14 @@ public class RollingLevelDBTimelineStore extends AbstractService implements try { value = fstConf.asObject(bytes); entity.addPrimaryFilter(name, value); - } catch (Exception e) { - LOG.warn("Error while decoding " + name, e); + } catch (Exception ignore) { + try { + // Fall back to 2.24 parser + value = fstConf224.asObject(bytes); + entity.addPrimaryFilter(name, value); + } catch (Exception e) { + LOG.warn("Error while decoding " + name, e); + } } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
