HDFS-10662. Optimize UTF8 string/byte conversions. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ae39199 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ae39199 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ae39199 Branch: refs/heads/YARN-2915 Commit: 6ae39199dac6ac7be6802b31452552c76da16e24 Parents: 70c2781 Author: Kihwal Lee <kih...@apache.org> Authored: Thu Aug 4 09:07:34 2016 -0500 Committer: Kihwal Lee <kih...@apache.org> Committed: Thu Aug 4 09:07:34 2016 -0500 ---------------------------------------------------------------------- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 22 ++++++++++++++------ .../java/org/apache/hadoop/hdfs/DFSUtil.java | 8 +------ .../hdfs/server/namenode/FSDirMkdirOp.java | 6 ++---- .../server/namenode/FSDirStatAndListingOp.java | 4 +--- .../hdfs/server/namenode/FSDirWriteFileOp.java | 4 ++-- .../hdfs/server/namenode/FSDirXAttrOp.java | 4 ++-- 6 files changed, 24 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ae39199/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java index 111c8c4..c511596 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java @@ -91,11 +91,21 @@ public class DFSUtilClient { public static final byte[] EMPTY_BYTES = {}; private static final Logger LOG = LoggerFactory.getLogger( DFSUtilClient.class); + + // Using the charset canonical name for String/byte[] conversions is much + // more efficient due to use of cached encoders/decoders. + private static final String UTF8_CSN = StandardCharsets.UTF_8.name(); + /** * Converts a string to a byte array using UTF8 encoding. */ public static byte[] string2Bytes(String str) { - return str.getBytes(StandardCharsets.UTF_8); + try { + return str.getBytes(UTF8_CSN); + } catch (UnsupportedEncodingException e) { + // should never happen! + throw new IllegalArgumentException("UTF8 decoding is not supported", e); + } } /** @@ -281,13 +291,13 @@ public class DFSUtilClient { * @param length The number of bytes to decode * @return The decoded string */ - private static String bytes2String(byte[] bytes, int offset, int length) { + static String bytes2String(byte[] bytes, int offset, int length) { try { - return new String(bytes, offset, length, "UTF8"); - } catch(UnsupportedEncodingException e) { - assert false : "UTF8 encoding is not supported "; + return new String(bytes, offset, length, UTF8_CSN); + } catch (UnsupportedEncodingException e) { + // should never happen! + throw new IllegalArgumentException("UTF8 encoding is not supported", e); } - return null; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ae39199/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java index 3fe944e..1ff39e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java @@ -38,7 +38,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_P import java.io.IOException; import java.io.PrintStream; -import java.io.UnsupportedEncodingException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.URI; @@ -248,12 +247,7 @@ public class DFSUtil { * @return The decoded string */ public static String bytes2String(byte[] bytes, int offset, int length) { - try { - return new String(bytes, offset, length, "UTF8"); - } catch(UnsupportedEncodingException e) { - assert false : "UTF8 encoding is not supported "; - } - return null; + return DFSUtilClient.bytes2String(bytes, 0, bytes.length); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ae39199/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java index ae73f9c..c74faca 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.List; import java.util.Map; @@ -121,8 +120,7 @@ class FSDirMkdirOp { static Map.Entry<INodesInPath, String> createAncestorDirectories( FSDirectory fsd, INodesInPath iip, PermissionStatus permission) throws IOException { - final String last = - new String(iip.getLastLocalName(), StandardCharsets.UTF_8); + final String last = DFSUtil.bytes2String(iip.getLastLocalName()); INodesInPath existing = iip.getExistingINodes(); List<String> children = iip.getPath(existing.length(), iip.length() - existing.length()); @@ -190,7 +188,7 @@ class FSDirMkdirOp { throws IOException { assert fsd.hasWriteLock(); existing = unprotectedMkdir(fsd, fsd.allocateNewInodeId(), existing, - localName.getBytes(StandardCharsets.UTF_8), perm, null, now()); + DFSUtil.string2Bytes(localName), perm, null, now()); if (existing == null) { return null; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ae39199/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java index 0604b9a..9fbc9f4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java @@ -45,7 +45,6 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import static org.apache.hadoop.util.Time.now; @@ -55,8 +54,7 @@ class FSDirStatAndListingOp { byte[] startAfter, boolean needLocation) throws IOException { byte[][] pathComponents = FSDirectory .getPathComponentsForReservedPath(srcArg); - final String startAfterString = - new String(startAfter, StandardCharsets.UTF_8); + final String startAfterString = DFSUtil.bytes2String(startAfter); String src = null; if (fsd.isPermissionEnabled()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ae39199/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java index d428c95..2e5893f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java @@ -32,6 +32,7 @@ import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.PermissionStatus; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy; import org.apache.hadoop.hdfs.protocol.ClientProtocol; @@ -59,7 +60,6 @@ import org.apache.hadoop.util.ChunkedArrayList; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -610,7 +610,7 @@ class FSDirWriteFileOp { } INodeFile newNode = newINodeFile(fsd.allocateNewInodeId(), permissions, modTime, modTime, replication, preferredBlockSize, ecPolicy != null); - newNode.setLocalName(localName.getBytes(StandardCharsets.UTF_8)); + newNode.setLocalName(DFSUtil.string2Bytes(localName)); newNode.toUnderConstruction(clientName, clientMachine); newiip = fsd.addINode(existing, newNode); } finally { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ae39199/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java index d27cec5..92686c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hdfs.server.namenode; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import org.apache.hadoop.HadoopIllegalArgumentException; @@ -26,6 +25,7 @@ import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.XAttrHelper; import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; @@ -410,7 +410,7 @@ class FSDirXAttrOp { * the configured limit. Setting a limit of zero disables this check. */ private static void checkXAttrSize(FSDirectory fsd, XAttr xAttr) { - int size = xAttr.getName().getBytes(Charsets.UTF_8).length; + int size = DFSUtil.string2Bytes(xAttr.getName()).length; if (xAttr.getValue() != null) { size += xAttr.getValue().length; } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
