http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java index 54e219b..fbd6eb8 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.client.rest; -import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo; +import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import java.util.List;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 43b94a1..fc70514 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -27,7 +27,7 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.ozone.KsmUtils; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; @@ -43,24 +43,22 @@ import org.apache.hadoop.ozone.client.io.LengthInputStream; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; -import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs; -import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; -import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; -import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs; -import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession; -import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo; -import org.apache.hadoop.ozone.ksm.protocolPB - .KeySpaceManagerProtocolClientSideTranslatorPB; -import org.apache.hadoop.ozone.ksm.protocolPB - .KeySpaceManagerProtocolPB; +import org.apache.hadoop.ozone.om.helpers.OmBucketArgs; +import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; +import org.apache.hadoop.ozone.om.helpers.OpenKeySession; +import org.apache.hadoop.ozone.om.helpers.ServiceInfo; +import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; +import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.ksm.KSMConfigKeys; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.ServicePort; + .OzoneManagerProtocolProtos.ServicePort; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; +import org.apache.hadoop.ozone.protocolPB.OMPBHelper; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.protocolPB @@ -80,7 +78,7 @@ import java.util.UUID; import java.util.stream.Collectors; /** - * Ozone RPC Client Implementation, it connects to KSM, SCM and DataNode + * Ozone RPC Client Implementation, it connects to OM, SCM and DataNode * to execute client calls. This uses RPC protocol for communication * with the servers. */ @@ -92,8 +90,8 @@ public class RpcClient implements ClientProtocol { private final OzoneConfiguration conf; private final StorageContainerLocationProtocolClientSideTranslatorPB storageContainerLocationClient; - private final KeySpaceManagerProtocolClientSideTranslatorPB - keySpaceManagerClient; + private final OzoneManagerProtocolClientSideTranslatorPB + ozoneManagerClient; private final XceiverClientManager xceiverClientManager; private final int chunkSize; private final UserGroupInformation ugi; @@ -109,20 +107,20 @@ public class RpcClient implements ClientProtocol { Preconditions.checkNotNull(conf); this.conf = new OzoneConfiguration(conf); this.ugi = UserGroupInformation.getCurrentUser(); - this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS, - KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT); - this.groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS, - KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT); - long ksmVersion = - RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class); - InetSocketAddress ksmAddress = KsmUtils - .getKsmAddressForClients(conf); - RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class, + this.userRights = conf.getEnum(OMConfigKeys.OZONE_OM_USER_RIGHTS, + OMConfigKeys.OZONE_OM_USER_RIGHTS_DEFAULT); + this.groupRights = conf.getEnum(OMConfigKeys.OZONE_OM_GROUP_RIGHTS, + OMConfigKeys.OZONE_OM_GROUP_RIGHTS_DEFAULT); + long omVersion = + RPC.getProtocolVersion(OzoneManagerProtocolPB.class); + InetSocketAddress omAddress = OmUtils + .getOmAddressForClients(conf); + RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class, ProtobufRpcEngine.class); - this.keySpaceManagerClient = - new KeySpaceManagerProtocolClientSideTranslatorPB( - RPC.getProxy(KeySpaceManagerProtocolPB.class, ksmVersion, - ksmAddress, UserGroupInformation.getCurrentUser(), conf, + this.ozoneManagerClient = + new OzoneManagerProtocolClientSideTranslatorPB( + RPC.getProxy(OzoneManagerProtocolPB.class, omVersion, + omAddress, UserGroupInformation.getCurrentUser(), conf, NetUtils.getDefaultSocketFactory(conf), Client.getRpcTimeout(conf))); @@ -155,7 +153,7 @@ public class RpcClient implements ClientProtocol { } private InetSocketAddress getScmAddressForClient() throws IOException { - List<ServiceInfo> services = keySpaceManagerClient.getServiceList(); + List<ServiceInfo> services = ozoneManagerClient.getServiceList(); ServiceInfo scmInfo = services.stream().filter( a -> a.getNodeType().equals(HddsProtos.NodeType.SCM)) .collect(Collectors.toList()).get(0); @@ -195,7 +193,7 @@ public class RpcClient implements ClientProtocol { listOfAcls.addAll(volArgs.getAcls()); } - KsmVolumeArgs.Builder builder = KsmVolumeArgs.newBuilder(); + OmVolumeArgs.Builder builder = OmVolumeArgs.newBuilder(); builder.setVolume(volumeName); builder.setAdminName(admin); builder.setOwnerName(owner); @@ -204,12 +202,12 @@ public class RpcClient implements ClientProtocol { //Remove duplicates and add ACLs for (OzoneAcl ozoneAcl : listOfAcls.stream().distinct().collect(Collectors.toList())) { - builder.addOzoneAcls(KSMPBHelper.convertOzoneAcl(ozoneAcl)); + builder.addOzoneAcls(OMPBHelper.convertOzoneAcl(ozoneAcl)); } LOG.info("Creating Volume: {}, with {} as owner and quota set to {} bytes.", volumeName, owner, quota); - keySpaceManagerClient.createVolume(builder.build()); + ozoneManagerClient.createVolume(builder.build()); } @Override @@ -217,7 +215,7 @@ public class RpcClient implements ClientProtocol { throws IOException { HddsClientUtils.verifyResourceName(volumeName); Preconditions.checkNotNull(owner); - keySpaceManagerClient.setOwner(volumeName, owner); + ozoneManagerClient.setOwner(volumeName, owner); } @Override @@ -226,14 +224,14 @@ public class RpcClient implements ClientProtocol { HddsClientUtils.verifyResourceName(volumeName); Preconditions.checkNotNull(quota); long quotaInBytes = quota.sizeInBytes(); - keySpaceManagerClient.setQuota(volumeName, quotaInBytes); + ozoneManagerClient.setQuota(volumeName, quotaInBytes); } @Override public OzoneVolume getVolumeDetails(String volumeName) throws IOException { HddsClientUtils.verifyResourceName(volumeName); - KsmVolumeArgs volume = keySpaceManagerClient.getVolumeInfo(volumeName); + OmVolumeArgs volume = ozoneManagerClient.getVolumeInfo(volumeName); return new OzoneVolume( conf, this, @@ -243,7 +241,7 @@ public class RpcClient implements ClientProtocol { volume.getQuotaInBytes(), volume.getCreationTime(), volume.getAclMap().ozoneAclGetProtobuf().stream(). - map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())); + map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList())); } @Override @@ -255,14 +253,14 @@ public class RpcClient implements ClientProtocol { @Override public void deleteVolume(String volumeName) throws IOException { HddsClientUtils.verifyResourceName(volumeName); - keySpaceManagerClient.deleteVolume(volumeName); + ozoneManagerClient.deleteVolume(volumeName); } @Override public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume, int maxListResult) throws IOException { - List<KsmVolumeArgs> volumes = keySpaceManagerClient.listAllVolumes( + List<OmVolumeArgs> volumes = ozoneManagerClient.listAllVolumes( volumePrefix, prevVolume, maxListResult); return volumes.stream().map(volume -> new OzoneVolume( @@ -274,7 +272,7 @@ public class RpcClient implements ClientProtocol { volume.getQuotaInBytes(), volume.getCreationTime(), volume.getAclMap().ozoneAclGetProtobuf().stream(). - map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))) + map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList()))) .collect(Collectors.toList()); } @@ -282,7 +280,7 @@ public class RpcClient implements ClientProtocol { public List<OzoneVolume> listVolumes(String user, String volumePrefix, String prevVolume, int maxListResult) throws IOException { - List<KsmVolumeArgs> volumes = keySpaceManagerClient.listVolumeByUser( + List<OmVolumeArgs> volumes = ozoneManagerClient.listVolumeByUser( user, volumePrefix, prevVolume, maxListResult); return volumes.stream().map(volume -> new OzoneVolume( @@ -294,7 +292,7 @@ public class RpcClient implements ClientProtocol { volume.getQuotaInBytes(), volume.getCreationTime(), volume.getAclMap().ozoneAclGetProtobuf().stream(). - map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))) + map(OMPBHelper::convertOzoneAcl).collect(Collectors.toList()))) .collect(Collectors.toList()); } @@ -329,7 +327,7 @@ public class RpcClient implements ClientProtocol { listOfAcls.addAll(bucketArgs.getAcls()); } - KsmBucketInfo.Builder builder = KsmBucketInfo.newBuilder(); + OmBucketInfo.Builder builder = OmBucketInfo.newBuilder(); builder.setVolumeName(volumeName) .setBucketName(bucketName) .setIsVersionEnabled(isVersionEnabled) @@ -339,7 +337,7 @@ public class RpcClient implements ClientProtocol { LOG.info("Creating Bucket: {}/{}, with Versioning {} and " + "Storage Type set to {}", volumeName, bucketName, isVersionEnabled, storageType); - keySpaceManagerClient.createBucket(builder.build()); + ozoneManagerClient.createBucket(builder.build()); } @Override @@ -348,11 +346,11 @@ public class RpcClient implements ClientProtocol { throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); Preconditions.checkNotNull(addAcls); - KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); builder.setVolumeName(volumeName) .setBucketName(bucketName) .setAddAcls(addAcls); - keySpaceManagerClient.setBucketProperty(builder.build()); + ozoneManagerClient.setBucketProperty(builder.build()); } @Override @@ -361,11 +359,11 @@ public class RpcClient implements ClientProtocol { throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); Preconditions.checkNotNull(removeAcls); - KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); builder.setVolumeName(volumeName) .setBucketName(bucketName) .setRemoveAcls(removeAcls); - keySpaceManagerClient.setBucketProperty(builder.build()); + ozoneManagerClient.setBucketProperty(builder.build()); } @Override @@ -374,11 +372,11 @@ public class RpcClient implements ClientProtocol { throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); Preconditions.checkNotNull(versioning); - KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); builder.setVolumeName(volumeName) .setBucketName(bucketName) .setIsVersionEnabled(versioning); - keySpaceManagerClient.setBucketProperty(builder.build()); + ozoneManagerClient.setBucketProperty(builder.build()); } @Override @@ -387,18 +385,18 @@ public class RpcClient implements ClientProtocol { throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); Preconditions.checkNotNull(storageType); - KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder(); + OmBucketArgs.Builder builder = OmBucketArgs.newBuilder(); builder.setVolumeName(volumeName) .setBucketName(bucketName) .setStorageType(storageType); - keySpaceManagerClient.setBucketProperty(builder.build()); + ozoneManagerClient.setBucketProperty(builder.build()); } @Override public void deleteBucket( String volumeName, String bucketName) throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); - keySpaceManagerClient.deleteBucket(volumeName, bucketName); + ozoneManagerClient.deleteBucket(volumeName, bucketName); } @Override @@ -411,8 +409,8 @@ public class RpcClient implements ClientProtocol { public OzoneBucket getBucketDetails( String volumeName, String bucketName) throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); - KsmBucketInfo bucketArgs = - keySpaceManagerClient.getBucketInfo(volumeName, bucketName); + OmBucketInfo bucketArgs = + ozoneManagerClient.getBucketInfo(volumeName, bucketName); return new OzoneBucket( conf, this, @@ -428,7 +426,7 @@ public class RpcClient implements ClientProtocol { public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix, String prevBucket, int maxListResult) throws IOException { - List<KsmBucketInfo> buckets = keySpaceManagerClient.listBuckets( + List<OmBucketInfo> buckets = ozoneManagerClient.listBuckets( volumeName, prevBucket, bucketPrefix, maxListResult); return buckets.stream().map(bucket -> new OzoneBucket( @@ -451,7 +449,7 @@ public class RpcClient implements ClientProtocol { HddsClientUtils.verifyResourceName(volumeName, bucketName); HddsClientUtils.checkNotNull(keyName, type, factor); String requestId = UUID.randomUUID().toString(); - KsmKeyArgs keyArgs = new KsmKeyArgs.Builder() + OmKeyArgs keyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(keyName) @@ -460,13 +458,13 @@ public class RpcClient implements ClientProtocol { .setFactor(HddsProtos.ReplicationFactor.valueOf(factor.getValue())) .build(); - OpenKeySession openKey = keySpaceManagerClient.openKey(keyArgs); + OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs); ChunkGroupOutputStream groupOutputStream = new ChunkGroupOutputStream.Builder() .setHandler(openKey) .setXceiverClientManager(xceiverClientManager) .setScmClient(storageContainerLocationClient) - .setKsmClient(keySpaceManagerClient) + .setOmClient(ozoneManagerClient) .setChunkSize(chunkSize) .setRequestID(requestId) .setType(HddsProtos.ReplicationType.valueOf(type.toString())) @@ -485,14 +483,14 @@ public class RpcClient implements ClientProtocol { HddsClientUtils.verifyResourceName(volumeName, bucketName); Preconditions.checkNotNull(keyName); String requestId = UUID.randomUUID().toString(); - KsmKeyArgs keyArgs = new KsmKeyArgs.Builder() + OmKeyArgs keyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(keyName) .build(); - KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs); + OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); LengthInputStream lengthInputStream = - ChunkGroupInputStream.getFromKsmKeyInfo( + ChunkGroupInputStream.getFromOmKeyInfo( keyInfo, xceiverClientManager, storageContainerLocationClient, requestId); return new OzoneInputStream( @@ -505,12 +503,12 @@ public class RpcClient implements ClientProtocol { throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); Preconditions.checkNotNull(keyName); - KsmKeyArgs keyArgs = new KsmKeyArgs.Builder() + OmKeyArgs keyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(keyName) .build(); - keySpaceManagerClient.deleteKey(keyArgs); + ozoneManagerClient.deleteKey(keyArgs); } @Override @@ -518,12 +516,12 @@ public class RpcClient implements ClientProtocol { String fromKeyName, String toKeyName) throws IOException { HddsClientUtils.verifyResourceName(volumeName, bucketName); HddsClientUtils.checkNotNull(fromKeyName, toKeyName); - KsmKeyArgs keyArgs = new KsmKeyArgs.Builder() + OmKeyArgs keyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(fromKeyName) .build(); - keySpaceManagerClient.renameKey(keyArgs, toKeyName); + ozoneManagerClient.renameKey(keyArgs, toKeyName); } @Override @@ -531,7 +529,7 @@ public class RpcClient implements ClientProtocol { String keyPrefix, String prevKey, int maxListResult) throws IOException { - List<KsmKeyInfo> keys = keySpaceManagerClient.listKeys( + List<OmKeyInfo> keys = ozoneManagerClient.listKeys( volumeName, bucketName, prevKey, keyPrefix, maxListResult); return keys.stream().map(key -> new OzoneKey( @@ -551,12 +549,12 @@ public class RpcClient implements ClientProtocol { Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); Preconditions.checkNotNull(keyName); - KsmKeyArgs keyArgs = new KsmKeyArgs.Builder() + OmKeyArgs keyArgs = new OmKeyArgs.Builder() .setVolumeName(volumeName) .setBucketName(bucketName) .setKeyName(keyName) .build(); - KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs); + OmKeyInfo keyInfo = ozoneManagerClient.lookupKey(keyArgs); return new OzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(), keyInfo.getKeyName(), @@ -568,7 +566,7 @@ public class RpcClient implements ClientProtocol { @Override public void close() throws IOException { IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient); - IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient); + IOUtils.cleanupWithLogger(LOG, ozoneManagerClient); IOUtils.cleanupWithLogger(LOG, xceiverClientManager); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java index a270f61..3aefe8a 100644 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java @@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.client; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.ksm.KSMConfigKeys; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.junit.Rule; import org.junit.Test; @@ -30,7 +30,7 @@ import org.junit.rules.Timeout; import java.net.InetSocketAddress; import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; -import static org.apache.hadoop.ozone.KsmUtils.getKsmAddress; +import static org.apache.hadoop.ozone.OmUtils.getOmAddress; import static org.hamcrest.core.Is.is; import static org.junit.Assert.assertThat; @@ -79,27 +79,27 @@ public class TestHddsClientUtils { } @Test - public void testGetKSMAddress() { + public void testGetOmAddress() { final Configuration conf = new OzoneConfiguration(); // First try a client address with just a host name. Verify it falls // back to the default port. - conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = getKsmAddress(conf); + conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4"); + InetSocketAddress addr = getOmAddress(conf); assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); + assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT)); // Next try a client address with just a host name and port. Verify the port - // is ignored and the default KSM port is used. - conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4:100"); - addr = getKsmAddress(conf); + // is ignored and the default OM port is used. + conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, "1.2.3.4:100"); + addr = getOmAddress(conf); assertThat(addr.getHostString(), is("1.2.3.4")); assertThat(addr.getPort(), is(100)); // Assert the we are able to use default configs if no value is specified. - conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, ""); - addr = getKsmAddress(conf); + conf.set(OMConfigKeys.OZONE_OM_ADDRESS_KEY, ""); + addr = getOmAddress(conf); assertThat(addr.getHostString(), is("0.0.0.0")); - assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); + assertThat(addr.getPort(), is(OMConfigKeys.OZONE_OM_PORT_DEFAULT)); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/pom.xml b/hadoop-ozone/common/pom.xml index d8581d1..83d023e 100644 --- a/hadoop-ozone/common/pom.xml +++ b/hadoop-ozone/common/pom.xml @@ -69,7 +69,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd";> <source> <directory>${basedir}/src/main/proto</directory> <includes> - <include>KeySpaceManagerProtocol.proto</include> + <include>OzoneManagerProtocol.proto</include> </includes> </source> </configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/bin/ozone ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone index 390f089..9495eff 100755 --- a/hadoop-ozone/common/src/main/bin/ozone +++ b/hadoop-ozone/common/src/main/bin/ozone @@ -38,10 +38,9 @@ function hadoop_usage hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables" hadoop_add_subcommand "freon" client "runs an ozone data generator" hadoop_add_subcommand "genesis" client "runs a collection of ozone benchmarks to help with tuning." - hadoop_add_subcommand "getozoneconf" client "get ozone config values from - configuration" + hadoop_add_subcommand "getozoneconf" client "get ozone config values from configuration" hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode." - hadoop_add_subcommand "ksm" daemon "Ozone keyspace manager" + hadoop_add_subcommand "om" daemon "Ozone Manager" hadoop_add_subcommand "o3" client "command line interface for ozone" hadoop_add_subcommand "noz" client "ozone debug tool, convert ozone metadata into relational data" hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service" @@ -94,9 +93,9 @@ function ozonecmd_case getozoneconf) HADOOP_CLASSNAME=org.apache.hadoop.ozone.freon.OzoneGetConf; ;; - ksm) + om) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME=org.apache.hadoop.ozone.ksm.KeySpaceManager + HADOOP_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManager ;; oz) HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/bin/start-ozone.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/start-ozone.sh b/hadoop-ozone/common/src/main/bin/start-ozone.sh index 92bc4a8..29c3674 100644 --- a/hadoop-ozone/common/src/main/bin/start-ozone.sh +++ b/hadoop-ozone/common/src/main/bin/start-ozone.sh @@ -179,19 +179,19 @@ if [[ "${AUTOHA_ENABLED}" = "true" ]]; then fi #--------------------------------------------------------- -# Ozone keyspacemanager nodes -KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -keyspacemanagers 2>/dev/null) -echo "Starting key space manager nodes [${KSM_NODES}]" -if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then - KSM_NODES=$(hostname) +# Ozone ozonemanager nodes +OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null) +echo "Starting Ozone Manager nodes [${OM_NODES}]" +if [[ "${OM_NODES}" == "0.0.0.0" ]]; then + OM_NODES=$(hostname) fi -hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/ozone" \ +hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \ --workers \ --config "${HADOOP_CONF_DIR}" \ - --hostnames "${KSM_NODES}" \ + --hostnames "${OM_NODES}" \ --daemon start \ - ksm + om HADOOP_JUMBO_RETCOUNTER=$? http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/bin/stop-ozone.sh ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/bin/stop-ozone.sh b/hadoop-ozone/common/src/main/bin/stop-ozone.sh index be55be4..5f5faf0 100644 --- a/hadoop-ozone/common/src/main/bin/stop-ozone.sh +++ b/hadoop-ozone/common/src/main/bin/stop-ozone.sh @@ -73,19 +73,19 @@ else fi #--------------------------------------------------------- -# Ozone keyspacemanager nodes -KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -keyspacemanagers 2>/dev/null) -echo "Stopping key space manager nodes [${KSM_NODES}]" -if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then - KSM_NODES=$(hostname) +# Ozone Manager nodes +OM_NODES=$("${HADOOP_HDFS_HOME}/bin/ozone" getozoneconf -ozonemanagers 2>/dev/null) +echo "Stopping Ozone Manager nodes [${OM_NODES}]" +if [[ "${OM_NODES}" == "0.0.0.0" ]]; then + OM_NODES=$(hostname) fi -hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/ozone" \ +hadoop_uservar_su hdfs om "${HADOOP_HDFS_HOME}/bin/ozone" \ --workers \ --config "${HADOOP_CONF_DIR}" \ - --hostnames "${KSM_NODES}" \ + --hostnames "${OM_NODES}" \ --daemon stop \ - ksm + om #--------------------------------------------------------- # Ozone storagecontainermanager nodes http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/KsmUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/KsmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/KsmUtils.java deleted file mode 100644 index 1025963..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/KsmUtils.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone; - -import java.net.InetSocketAddress; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.net.NetUtils; - -import com.google.common.base.Optional; -import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; -import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; -import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY; -import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY; -import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_HTTP_BIND_PORT_DEFAULT; -import static org.apache.hadoop.ozone.ksm.KSMConfigKeys - .OZONE_KSM_BIND_HOST_DEFAULT; -import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT; - -/** - * Stateless helper functions for the server and client side of KSM - * communication. - */ -public final class KsmUtils { - - private KsmUtils() { - } - - /** - * Retrieve the socket address that is used by KSM. - * @param conf - * @return Target InetSocketAddress for the SCM service endpoint. - */ - public static InetSocketAddress getKsmAddress( - Configuration conf) { - final Optional<String> host = getHostNameFromConfigKeys(conf, - OZONE_KSM_ADDRESS_KEY); - - return NetUtils.createSocketAddr( - host.or(OZONE_KSM_BIND_HOST_DEFAULT) + ":" + - getKsmRpcPort(conf)); - } - - /** - * Retrieve the socket address that should be used by clients to connect - * to KSM. - * @param conf - * @return Target InetSocketAddress for the KSM service endpoint. - */ - public static InetSocketAddress getKsmAddressForClients( - Configuration conf) { - final Optional<String> host = getHostNameFromConfigKeys(conf, - OZONE_KSM_ADDRESS_KEY); - - if (!host.isPresent()) { - throw new IllegalArgumentException( - OZONE_KSM_ADDRESS_KEY + " must be defined. See" + - " https://wiki.apache.org/hadoop/Ozone#Configuration for" + - " details on configuring Ozone."); - } - - return NetUtils.createSocketAddr( - host.get() + ":" + getKsmRpcPort(conf)); - } - - public static int getKsmRpcPort(Configuration conf) { - // If no port number is specified then we'll just try the defaultBindPort. - final Optional<Integer> port = getPortNumberFromConfigKeys(conf, - OZONE_KSM_ADDRESS_KEY); - return port.or(OZONE_KSM_PORT_DEFAULT); - } - - public static int getKsmRestPort(Configuration conf) { - // If no port number is specified then we'll just try the default - // HTTP BindPort. - final Optional<Integer> port = - getPortNumberFromConfigKeys(conf, OZONE_KSM_HTTP_ADDRESS_KEY); - return port.or(OZONE_KSM_HTTP_BIND_PORT_DEFAULT); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java new file mode 100644 index 0000000..0974104 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone; + +import java.net.InetSocketAddress; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.net.NetUtils; + +import com.google.common.base.Optional; +import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; +import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_BIND_HOST_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_BIND_PORT_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_PORT_DEFAULT; + +/** + * Stateless helper functions for the server and client side of OM + * communication. + */ +public final class OmUtils { + + private OmUtils() { + } + + /** + * Retrieve the socket address that is used by OM. + * @param conf + * @return Target InetSocketAddress for the SCM service endpoint. + */ + public static InetSocketAddress getOmAddress( + Configuration conf) { + final Optional<String> host = getHostNameFromConfigKeys(conf, + OZONE_OM_ADDRESS_KEY); + + return NetUtils.createSocketAddr( + host.or(OZONE_OM_BIND_HOST_DEFAULT) + ":" + + getOmRpcPort(conf)); + } + + /** + * Retrieve the socket address that should be used by clients to connect + * to OM. + * @param conf + * @return Target InetSocketAddress for the OM service endpoint. + */ + public static InetSocketAddress getOmAddressForClients( + Configuration conf) { + final Optional<String> host = getHostNameFromConfigKeys(conf, + OZONE_OM_ADDRESS_KEY); + + if (!host.isPresent()) { + throw new IllegalArgumentException( + OZONE_OM_ADDRESS_KEY + " must be defined. See" + + " https://wiki.apache.org/hadoop/Ozone#Configuration for" + + " details on configuring Ozone."); + } + + return NetUtils.createSocketAddr( + host.get() + ":" + getOmRpcPort(conf)); + } + + public static int getOmRpcPort(Configuration conf) { + // If no port number is specified then we'll just try the defaultBindPort. + final Optional<Integer> port = getPortNumberFromConfigKeys(conf, + OZONE_OM_ADDRESS_KEY); + return port.or(OZONE_OM_PORT_DEFAULT); + } + + public static int getOmRestPort(Configuration conf) { + // If no port number is specified then we'll just try the default + // HTTP BindPort. + final Optional<Integer> port = + getPortNumberFromConfigKeys(conf, OZONE_OM_HTTP_ADDRESS_KEY); + return port.or(OZONE_OM_HTTP_BIND_PORT_DEFAULT); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java index d5f9093..ffbca6a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java @@ -32,7 +32,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.ozone.KsmUtils; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; @@ -53,8 +53,8 @@ public class OzoneGetConf extends Configured implements Tool { EXCLUDE_FILE("-excludeFile", "gets the exclude file path that defines the datanodes " + "that need to decommissioned."), - KEYSPACEMANAGER("-keyspacemanagers", - "gets list of ozone key space manager nodes in the cluster"), + OZONEMANAGER("-ozonemanagers", + "gets list of Ozone Manager nodes in the cluster"), STORAGECONTAINERMANAGER("-storagecontainermanagers", "gets list of ozone storage container manager nodes in the cluster"), CONFKEY("-confKey [key]", "gets a specific key from the configuration"); @@ -63,8 +63,8 @@ public class OzoneGetConf extends Configured implements Tool { static { HANDLERS = new HashMap<String, OzoneGetConf.CommandHandler>(); - HANDLERS.put(StringUtils.toLowerCase(KEYSPACEMANAGER.getName()), - new KeySpaceManagersCommandHandler()); + HANDLERS.put(StringUtils.toLowerCase(OZONEMANAGER.getName()), + new OzoneManagersCommandHandler()); HANDLERS.put(StringUtils.toLowerCase(STORAGECONTAINERMANAGER.getName()), new StorageContainerManagersCommandHandler()); HANDLERS.put(StringUtils.toLowerCase(CONFKEY.getName()), @@ -245,13 +245,13 @@ public class OzoneGetConf extends Configured implements Tool { } /** - * Handler for {@link Command#KEYSPACEMANAGER}. + * Handler for {@link Command#OZONEMANAGER}. */ - static class KeySpaceManagersCommandHandler extends CommandHandler { + static class OzoneManagersCommandHandler extends CommandHandler { @Override public int doWorkInternal(OzoneGetConf tool, String[] args) throws IOException { - tool.printOut(KsmUtils.getKsmAddress(tool.getConf()).getHostName()); + tool.printOut(OmUtils.getOmAddress(tool.getConf()).getHostName()); return 0; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java deleted file mode 100644 index 75cf613..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.ksm; - -import org.apache.hadoop.ozone.OzoneAcl; -/** - * KSM Constants. - */ -public final class KSMConfigKeys { - /** - * Never constructed. - */ - private KSMConfigKeys() { - } - - - public static final String OZONE_KSM_HANDLER_COUNT_KEY = - "ozone.ksm.handler.count.key"; - public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20; - - public static final String OZONE_KSM_ADDRESS_KEY = - "ozone.ksm.address"; - public static final String OZONE_KSM_BIND_HOST_DEFAULT = - "0.0.0.0"; - public static final int OZONE_KSM_PORT_DEFAULT = 9862; - - public static final String OZONE_KSM_HTTP_ENABLED_KEY = - "ozone.ksm.http.enabled"; - public static final String OZONE_KSM_HTTP_BIND_HOST_KEY = - "ozone.ksm.http-bind-host"; - public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY = - "ozone.ksm.https-bind-host"; - public static final String OZONE_KSM_HTTP_ADDRESS_KEY = - "ozone.ksm.http-address"; - public static final String OZONE_KSM_HTTPS_ADDRESS_KEY = - "ozone.ksm.https-address"; - public static final String OZONE_KSM_KEYTAB_FILE = - "ozone.ksm.keytab.file"; - public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0"; - public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874; - public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875; - - // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB. - public static final String OZONE_KSM_DB_CACHE_SIZE_MB = - "ozone.ksm.db.cache.size.mb"; - public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128; - - public static final String OZONE_KSM_USER_MAX_VOLUME = - "ozone.ksm.user.max.volume"; - public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024; - - // KSM Default user/group permissions - public static final String OZONE_KSM_USER_RIGHTS = - "ozone.ksm.user.rights"; - public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT = - OzoneAcl.OzoneACLRights.READ_WRITE; - - public static final String OZONE_KSM_GROUP_RIGHTS = - "ozone.ksm.group.rights"; - public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT = - OzoneAcl.OzoneACLRights.READ_WRITE; - - public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK = - "ozone.key.deleting.limit.per.task"; - public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java deleted file mode 100644 index 1211b50..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.ksm.helpers; - -import java.util.List; -import java.util.stream.Collectors; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.BucketArgs; -import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; - -/** - * A class that encapsulates Bucket Arguments. - */ -public final class KsmBucketArgs { - /** - * Name of the volume in which the bucket belongs to. - */ - private final String volumeName; - /** - * Name of the bucket. - */ - private final String bucketName; - /** - * ACL's that are to be added for the bucket. - */ - private List<OzoneAcl> addAcls; - /** - * ACL's that are to be removed from the bucket. - */ - private List<OzoneAcl> removeAcls; - /** - * Bucket Version flag. - */ - private Boolean isVersionEnabled; - /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] - */ - private StorageType storageType; - - /** - * Private constructor, constructed via builder. - * @param volumeName - Volume name. - * @param bucketName - Bucket name. - * @param addAcls - ACL's to be added. - * @param removeAcls - ACL's to be removed. - * @param isVersionEnabled - Bucket version flag. - * @param storageType - Storage type to be used. - */ - private KsmBucketArgs(String volumeName, String bucketName, - List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls, - Boolean isVersionEnabled, StorageType storageType) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.addAcls = addAcls; - this.removeAcls = removeAcls; - this.isVersionEnabled = isVersionEnabled; - this.storageType = storageType; - } - - /** - * Returns the Volume Name. - * @return String. - */ - public String getVolumeName() { - return volumeName; - } - - /** - * Returns the Bucket Name. - * @return String - */ - public String getBucketName() { - return bucketName; - } - - /** - * Returns the ACL's that are to be added. - * @return List<OzoneAclInfo> - */ - public List<OzoneAcl> getAddAcls() { - return addAcls; - } - - /** - * Returns the ACL's that are to be removed. - * @return List<OzoneAclInfo> - */ - public List<OzoneAcl> getRemoveAcls() { - return removeAcls; - } - - /** - * Returns true if bucket version is enabled, else false. - * @return isVersionEnabled - */ - public Boolean getIsVersionEnabled() { - return isVersionEnabled; - } - - /** - * Returns the type of storage to be used. - * @return StorageType - */ - public StorageType getStorageType() { - return storageType; - } - - /** - * Returns new builder class that builds a KsmBucketArgs. - * - * @return Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder for KsmBucketArgs. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private List<OzoneAcl> addAcls; - private List<OzoneAcl> removeAcls; - private Boolean isVersionEnabled; - private StorageType storageType; - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setAddAcls(List<OzoneAcl> acls) { - this.addAcls = acls; - return this; - } - - public Builder setRemoveAcls(List<OzoneAcl> acls) { - this.removeAcls = acls; - return this; - } - - public Builder setIsVersionEnabled(Boolean versionFlag) { - this.isVersionEnabled = versionFlag; - return this; - } - - public Builder setStorageType(StorageType storage) { - this.storageType = storage; - return this; - } - - /** - * Constructs the KsmBucketArgs. - * @return instance of KsmBucketArgs. - */ - public KsmBucketArgs build() { - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - return new KsmBucketArgs(volumeName, bucketName, addAcls, - removeAcls, isVersionEnabled, storageType); - } - } - - /** - * Creates BucketArgs protobuf from KsmBucketArgs. - */ - public BucketArgs getProtobuf() { - BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setVolumeName(volumeName) - .setBucketName(bucketName); - if(addAcls != null && !addAcls.isEmpty()) { - builder.addAllAddAcls(addAcls.stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())); - } - if(removeAcls != null && !removeAcls.isEmpty()) { - builder.addAllRemoveAcls(removeAcls.stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())); - } - if(isVersionEnabled != null) { - builder.setIsVersionEnabled(isVersionEnabled); - } - if(storageType != null) { - builder.setStorageType( - PBHelperClient.convertStorageType(storageType)); - } - return builder.build(); - } - - /** - * Parses BucketInfo protobuf and creates KsmBucketArgs. - * @param bucketArgs - * @return instance of KsmBucketArgs - */ - public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) { - return new KsmBucketArgs(bucketArgs.getVolumeName(), - bucketArgs.getBucketName(), - bucketArgs.getAddAclsList().stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), - bucketArgs.getRemoveAclsList().stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), - bucketArgs.hasIsVersionEnabled() ? - bucketArgs.getIsVersionEnabled() : null, - bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType( - bucketArgs.getStorageType()) : null); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java deleted file mode 100644 index a49137a..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java +++ /dev/null @@ -1,235 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.ksm.helpers; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.BucketInfo; -import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; - -import java.util.LinkedList; -import java.util.List; -import java.util.stream.Collectors; - -/** - * A class that encapsulates Bucket Info. - */ -public final class KsmBucketInfo { - /** - * Name of the volume in which the bucket belongs to. - */ - private final String volumeName; - /** - * Name of the bucket. - */ - private final String bucketName; - /** - * ACL Information. - */ - private List<OzoneAcl> acls; - /** - * Bucket Version flag. - */ - private Boolean isVersionEnabled; - /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] - */ - private StorageType storageType; - /** - * Creation time of bucket. - */ - private final long creationTime; - - /** - * Private constructor, constructed via builder. - * @param volumeName - Volume name. - * @param bucketName - Bucket name. - * @param acls - list of ACLs. - * @param isVersionEnabled - Bucket version flag. - * @param storageType - Storage type to be used. - * @param creationTime - Bucket creation time. - */ - private KsmBucketInfo(String volumeName, String bucketName, - List<OzoneAcl> acls, boolean isVersionEnabled, - StorageType storageType, long creationTime) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.acls = acls; - this.isVersionEnabled = isVersionEnabled; - this.storageType = storageType; - this.creationTime = creationTime; - } - - /** - * Returns the Volume Name. - * @return String. - */ - public String getVolumeName() { - return volumeName; - } - - /** - * Returns the Bucket Name. - * @return String - */ - public String getBucketName() { - return bucketName; - } - - /** - * Returns the ACL's associated with this bucket. - * @return List<OzoneAcl> - */ - public List<OzoneAcl> getAcls() { - return acls; - } - - /** - * Returns true if bucket version is enabled, else false. - * @return isVersionEnabled - */ - public boolean getIsVersionEnabled() { - return isVersionEnabled; - } - - /** - * Returns the type of storage to be used. - * @return StorageType - */ - public StorageType getStorageType() { - return storageType; - } - - /** - * Returns creation time. - * - * @return long - */ - public long getCreationTime() { - return creationTime; - } - - /** - * Returns new builder class that builds a KsmBucketInfo. - * - * @return Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder for KsmBucketInfo. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private List<OzoneAcl> acls; - private Boolean isVersionEnabled; - private StorageType storageType; - private long creationTime; - - Builder() { - //Default values - this.acls = new LinkedList<>(); - this.isVersionEnabled = false; - this.storageType = StorageType.DISK; - } - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setAcls(List<OzoneAcl> listOfAcls) { - this.acls = listOfAcls; - return this; - } - - public Builder setIsVersionEnabled(Boolean versionFlag) { - this.isVersionEnabled = versionFlag; - return this; - } - - public Builder setStorageType(StorageType storage) { - this.storageType = storage; - return this; - } - - public Builder setCreationTime(long createdOn) { - this.creationTime = createdOn; - return this; - } - - /** - * Constructs the KsmBucketInfo. - * @return instance of KsmBucketInfo. - */ - public KsmBucketInfo build() { - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - Preconditions.checkNotNull(acls); - Preconditions.checkNotNull(isVersionEnabled); - Preconditions.checkNotNull(storageType); - - return new KsmBucketInfo(volumeName, bucketName, acls, - isVersionEnabled, storageType, creationTime); - } - } - - /** - * Creates BucketInfo protobuf from KsmBucketInfo. - */ - public BucketInfo getProtobuf() { - return BucketInfo.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .addAllAcls(acls.stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())) - .setIsVersionEnabled(isVersionEnabled) - .setStorageType(PBHelperClient.convertStorageType( - storageType)) - .setCreationTime(creationTime) - .build(); - } - - /** - * Parses BucketInfo protobuf and creates KsmBucketInfo. - * @param bucketInfo - * @return instance of KsmBucketInfo - */ - public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) { - return new KsmBucketInfo( - bucketInfo.getVolumeName(), - bucketInfo.getBucketName(), - bucketInfo.getAclsList().stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), - bucketInfo.getIsVersionEnabled(), - PBHelperClient.convertStorageType( - bucketInfo.getStorageType()), bucketInfo.getCreationTime()); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java deleted file mode 100644 index cd17e28..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java +++ /dev/null @@ -1,119 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.ksm.helpers; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; - -/** - * Args for key. Client use this to specify key's attributes on key creation - * (putKey()). - */ -public final class KsmKeyArgs { - private final String volumeName; - private final String bucketName; - private final String keyName; - private long dataSize; - private final ReplicationType type; - private final ReplicationFactor factor; - - private KsmKeyArgs(String volumeName, String bucketName, String keyName, - long dataSize, ReplicationType type, ReplicationFactor factor) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.keyName = keyName; - this.dataSize = dataSize; - this.type = type; - this.factor = factor; - } - - public ReplicationType getType() { - return type; - } - - public ReplicationFactor getFactor() { - return factor; - } - - public String getVolumeName() { - return volumeName; - } - - public String getBucketName() { - return bucketName; - } - - public String getKeyName() { - return keyName; - } - - public long getDataSize() { - return dataSize; - } - - public void setDataSize(long size) { - dataSize = size; - } - - /** - * Builder class of KsmKeyArgs. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private String keyName; - private long dataSize; - private ReplicationType type; - private ReplicationFactor factor; - - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setKeyName(String key) { - this.keyName = key; - return this; - } - - public Builder setDataSize(long size) { - this.dataSize = size; - return this; - } - - public Builder setType(ReplicationType replicationType) { - this.type = replicationType; - return this; - } - - public Builder setFactor(ReplicationFactor replicationFactor) { - this.factor = replicationFactor; - return this; - } - - public KsmKeyArgs build() { - return new KsmKeyArgs(volumeName, bucketName, keyName, dataSize, - type, factor); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java deleted file mode 100644 index 5d6e633..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.ksm.helpers; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.protocol.proto.HddsProtos; -import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo; -import org.apache.hadoop.util.Time; - -import java.io.IOException; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Args for key block. The block instance for the key requested in putKey. - * This is returned from KSM to client, and client use class to talk to - * datanode. Also, this is the metadata written to ksm.db on server side. - */ -public final class KsmKeyInfo { - private final String volumeName; - private final String bucketName; - // name of key client specified - private String keyName; - private long dataSize; - private List<KsmKeyLocationInfoGroup> keyLocationVersions; - private final long creationTime; - private long modificationTime; - private HddsProtos.ReplicationType type; - private HddsProtos.ReplicationFactor factor; - - private KsmKeyInfo(String volumeName, String bucketName, String keyName, - List<KsmKeyLocationInfoGroup> versions, long dataSize, - long creationTime, long modificationTime, HddsProtos.ReplicationType type, - HddsProtos.ReplicationFactor factor) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.keyName = keyName; - this.dataSize = dataSize; - // it is important that the versions are ordered from old to new. - // Do this sanity check when versions got loaded on creating KsmKeyInfo. - // TODO : this is not necessary, here only because versioning is still a - // work in-progress, remove this following check when versioning is - // complete and prove correctly functioning - long currentVersion = -1; - for (KsmKeyLocationInfoGroup version : versions) { - Preconditions.checkArgument( - currentVersion + 1 == version.getVersion()); - currentVersion = version.getVersion(); - } - this.keyLocationVersions = versions; - this.creationTime = creationTime; - this.modificationTime = modificationTime; - this.factor = factor; - this.type = type; - } - - public String getVolumeName() { - return volumeName; - } - - public String getBucketName() { - return bucketName; - } - - public HddsProtos.ReplicationType getType() { - return type; - } - - public HddsProtos.ReplicationFactor getFactor() { - return factor; - } - - public String getKeyName() { - return keyName; - } - - public void setKeyName(String keyName) { - this.keyName = keyName; - } - - public long getDataSize() { - return dataSize; - } - - public void setDataSize(long size) { - this.dataSize = size; - } - - public synchronized KsmKeyLocationInfoGroup getLatestVersionLocations() - throws IOException { - return keyLocationVersions.size() == 0? null : - keyLocationVersions.get(keyLocationVersions.size() - 1); - } - - public List<KsmKeyLocationInfoGroup> getKeyLocationVersions() { - return keyLocationVersions; - } - - public void updateModifcationTime() { - this.modificationTime = Time.monotonicNow(); - } - - /** - * Append a set of blocks to the latest version. Note that these blocks are - * part of the latest version, not a new version. - * - * @param newLocationList the list of new blocks to be added. - * @throws IOException - */ - public synchronized void appendNewBlocks( - List<KsmKeyLocationInfo> newLocationList) throws IOException { - if (keyLocationVersions.size() == 0) { - throw new IOException("Appending new block, but no version exist"); - } - KsmKeyLocationInfoGroup currentLatestVersion = - keyLocationVersions.get(keyLocationVersions.size() - 1); - currentLatestVersion.appendNewBlocks(newLocationList); - setModificationTime(Time.now()); - } - - /** - * Add a new set of blocks. The new blocks will be added as appending a new - * version to the all version list. - * - * @param newLocationList the list of new blocks to be added. - * @throws IOException - */ - public synchronized long addNewVersion( - List<KsmKeyLocationInfo> newLocationList) throws IOException { - long latestVersionNum; - if (keyLocationVersions.size() == 0) { - // no version exist, these blocks are the very first version. - keyLocationVersions.add(new KsmKeyLocationInfoGroup(0, newLocationList)); - latestVersionNum = 0; - } else { - // it is important that the new version are always at the tail of the list - KsmKeyLocationInfoGroup currentLatestVersion = - keyLocationVersions.get(keyLocationVersions.size() - 1); - // the new version is created based on the current latest version - KsmKeyLocationInfoGroup newVersion = - currentLatestVersion.generateNextVersion(newLocationList); - keyLocationVersions.add(newVersion); - latestVersionNum = newVersion.getVersion(); - } - setModificationTime(Time.now()); - return latestVersionNum; - } - - public long getCreationTime() { - return creationTime; - } - - public long getModificationTime() { - return modificationTime; - } - - public void setModificationTime(long modificationTime) { - this.modificationTime = modificationTime; - } - - /** - * Builder of KsmKeyInfo. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private String keyName; - private long dataSize; - private List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups; - private long creationTime; - private long modificationTime; - private HddsProtos.ReplicationType type; - private HddsProtos.ReplicationFactor factor; - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setKeyName(String key) { - this.keyName = key; - return this; - } - - public Builder setKsmKeyLocationInfos( - List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoList) { - this.ksmKeyLocationInfoGroups = ksmKeyLocationInfoList; - return this; - } - - public Builder setDataSize(long size) { - this.dataSize = size; - return this; - } - - public Builder setCreationTime(long crTime) { - this.creationTime = crTime; - return this; - } - - public Builder setModificationTime(long mTime) { - this.modificationTime = mTime; - return this; - } - - public Builder setReplicationFactor(HddsProtos.ReplicationFactor factor) { - this.factor = factor; - return this; - } - - public Builder setReplicationType(HddsProtos.ReplicationType type) { - this.type = type; - return this; - } - - public KsmKeyInfo build() { - return new KsmKeyInfo( - volumeName, bucketName, keyName, ksmKeyLocationInfoGroups, - dataSize, creationTime, modificationTime, type, factor); - } - } - - public KeyInfo getProtobuf() { - long latestVersion = keyLocationVersions.size() == 0 ? -1 : - keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion(); - return KeyInfo.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(dataSize) - .setFactor(factor) - .setType(type) - .addAllKeyLocationList(keyLocationVersions.stream() - .map(KsmKeyLocationInfoGroup::getProtobuf) - .collect(Collectors.toList())) - .setLatestVersion(latestVersion) - .setCreationTime(creationTime) - .setModificationTime(modificationTime) - .build(); - } - - public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) { - return new KsmKeyInfo( - keyInfo.getVolumeName(), - keyInfo.getBucketName(), - keyInfo.getKeyName(), - keyInfo.getKeyLocationListList().stream() - .map(KsmKeyLocationInfoGroup::getFromProtobuf) - .collect(Collectors.toList()), - keyInfo.getDataSize(), - keyInfo.getCreationTime(), - keyInfo.getModificationTime(), - keyInfo.getType(), - keyInfo.getFactor()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java deleted file mode 100644 index 45feda0..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.ksm.helpers; - -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocation; - -/** - * One key can be too huge to fit in one container. In which case it gets split - * into a number of subkeys. This class represents one such subkey instance. - */ -public final class KsmKeyLocationInfo { - private final BlockID blockID; - private final boolean shouldCreateContainer; - // the id of this subkey in all the subkeys. - private final long length; - private final long offset; - // the version number indicating when this block was added - private long createVersion; - - private KsmKeyLocationInfo(BlockID blockID, boolean shouldCreateContainer, - long length, long offset) { - this.blockID = blockID; - this.shouldCreateContainer = shouldCreateContainer; - this.length = length; - this.offset = offset; - } - - public void setCreateVersion(long version) { - createVersion = version; - } - - public long getCreateVersion() { - return createVersion; - } - - public BlockID getBlockID() { - return blockID; - } - - public long getContainerID() { - return blockID.getContainerID(); - } - - public long getLocalID() { - return blockID.getLocalID(); - } - - public boolean getShouldCreateContainer() { - return shouldCreateContainer; - } - - public long getLength() { - return length; - } - - public long getOffset() { - return offset; - } - - /** - * Builder of KsmKeyLocationInfo. - */ - public static class Builder { - private BlockID blockID; - private boolean shouldCreateContainer; - private long length; - private long offset; - - public Builder setBlockID(BlockID blockId) { - this.blockID = blockId; - return this; - } - - public Builder setShouldCreateContainer(boolean create) { - this.shouldCreateContainer = create; - return this; - } - - public Builder setLength(long len) { - this.length = len; - return this; - } - - public Builder setOffset(long off) { - this.offset = off; - return this; - } - - public KsmKeyLocationInfo build() { - return new KsmKeyLocationInfo(blockID, - shouldCreateContainer, length, offset); - } - } - - public KeyLocation getProtobuf() { - return KeyLocation.newBuilder() - .setBlockID(blockID.getProtobuf()) - .setShouldCreateContainer(shouldCreateContainer) - .setLength(length) - .setOffset(offset) - .setCreateVersion(createVersion) - .build(); - } - - public static KsmKeyLocationInfo getFromProtobuf(KeyLocation keyLocation) { - KsmKeyLocationInfo info = new KsmKeyLocationInfo( - BlockID.getFromProtobuf(keyLocation.getBlockID()), - keyLocation.getShouldCreateContainer(), - keyLocation.getLength(), - keyLocation.getOffset()); - info.setCreateVersion(keyLocation.getCreateVersion()); - return info; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java deleted file mode 100644 index 0facf3c..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java +++ /dev/null @@ -1,118 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ozone.ksm.helpers; - -import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocationList; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -/** - * A list of key locations. This class represents one single version of the - * blocks of a key. - */ -public class KsmKeyLocationInfoGroup { - private final long version; - private final List<KsmKeyLocationInfo> locationList; - - public KsmKeyLocationInfoGroup(long version, - List<KsmKeyLocationInfo> locations) { - this.version = version; - this.locationList = locations; - } - - /** - * Return only the blocks that are created in the most recent version. - * - * @return the list of blocks that are created in the latest version. - */ - public List<KsmKeyLocationInfo> getBlocksLatestVersionOnly() { - List<KsmKeyLocationInfo> list = new ArrayList<>(); - locationList.stream().filter(x -> x.getCreateVersion() == version) - .forEach(list::add); - return list; - } - - public long getVersion() { - return version; - } - - public List<KsmKeyLocationInfo> getLocationList() { - return locationList; - } - - public KeyLocationList getProtobuf() { - return KeyLocationList.newBuilder() - .setVersion(version) - .addAllKeyLocations( - locationList.stream().map(KsmKeyLocationInfo::getProtobuf) - .collect(Collectors.toList())) - .build(); - } - - public static KsmKeyLocationInfoGroup getFromProtobuf( - KeyLocationList keyLocationList) { - return new KsmKeyLocationInfoGroup( - keyLocationList.getVersion(), - keyLocationList.getKeyLocationsList().stream() - .map(KsmKeyLocationInfo::getFromProtobuf) - .collect(Collectors.toList())); - } - - /** - * Given a new block location, generate a new version list based upon this - * one. - * - * @param newLocationList a list of new location to be added. - * @return - */ - KsmKeyLocationInfoGroup generateNextVersion( - List<KsmKeyLocationInfo> newLocationList) throws IOException { - // TODO : revisit if we can do this method more efficiently - // one potential inefficiency here is that later version always include - // older ones. e.g. v1 has B1, then v2, v3...will all have B1 and only add - // more - List<KsmKeyLocationInfo> newList = new ArrayList<>(); - newList.addAll(locationList); - for (KsmKeyLocationInfo newInfo : newLocationList) { - // all these new blocks will have addVersion of current version + 1 - newInfo.setCreateVersion(version + 1); - newList.add(newInfo); - } - return new KsmKeyLocationInfoGroup(version + 1, newList); - } - - void appendNewBlocks(List<KsmKeyLocationInfo> newLocationList) - throws IOException { - for (KsmKeyLocationInfo info : newLocationList) { - info.setCreateVersion(version); - locationList.add(info); - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("version:").append(version).append(" "); - for (KsmKeyLocationInfo kli : locationList) { - sb.append(kli.getLocalID()).append(" || "); - } - return sb.toString(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/061b1685/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java deleted file mode 100644 index 7d9efad..0000000 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.ksm.helpers; - -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType; - -import java.util.List; -import java.util.LinkedList; -import java.util.Map; -import java.util.ArrayList; -import java.util.HashMap; - -/** - * This helper class keeps a map of all user and their permissions. - */ -public class KsmOzoneAclMap { - // per Acl Type user:rights map - private ArrayList<Map<String, OzoneAclRights>> aclMaps; - - KsmOzoneAclMap() { - aclMaps = new ArrayList<>(); - for (OzoneAclType aclType : OzoneAclType.values()) { - aclMaps.add(aclType.ordinal(), new HashMap<>()); - } - } - - private Map<String, OzoneAclRights> getMap(OzoneAclType type) { - return aclMaps.get(type.ordinal()); - } - - // For a given acl type and user, get the stored acl - private OzoneAclRights getAcl(OzoneAclType type, String user) { - return getMap(type).get(user); - } - - // Add a new acl to the map - public void addAcl(OzoneAclInfo acl) { - getMap(acl.getType()).put(acl.getName(), acl.getRights()); - } - - // for a given acl, check if the user has access rights - public boolean hasAccess(OzoneAclInfo acl) { - OzoneAclRights storedRights = getAcl(acl.getType(), acl.getName()); - if (storedRights != null) { - switch (acl.getRights()) { - case READ: - return (storedRights == OzoneAclRights.READ) - || (storedRights == OzoneAclRights.READ_WRITE); - case WRITE: - return (storedRights == OzoneAclRights.WRITE) - || (storedRights == OzoneAclRights.READ_WRITE); - case READ_WRITE: - return (storedRights == OzoneAclRights.READ_WRITE); - default: - return false; - } - } else { - return false; - } - } - - // Convert this map to OzoneAclInfo Protobuf List - public List<OzoneAclInfo> ozoneAclGetProtobuf() { - List<OzoneAclInfo> aclList = new LinkedList<>(); - for (OzoneAclType type: OzoneAclType.values()) { - for (Map.Entry<String, OzoneAclRights> entry : - aclMaps.get(type.ordinal()).entrySet()) { - OzoneAclInfo aclInfo = OzoneAclInfo.newBuilder() - .setName(entry.getKey()) - .setType(type) - .setRights(entry.getValue()) - .build(); - aclList.add(aclInfo); - } - } - - return aclList; - } - - // Create map from list of OzoneAclInfos - public static KsmOzoneAclMap ozoneAclGetFromProtobuf( - List<OzoneAclInfo> aclList) { - KsmOzoneAclMap aclMap = new KsmOzoneAclMap(); - for (OzoneAclInfo acl : aclList) { - aclMap.addAcl(acl); - } - return aclMap; - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
