http://git-wip-us.apache.org/repos/asf/hadoop/blob/df21e1b1/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java index 2200cd8..f56cbe8 100644 --- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java +++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java @@ -1,64 +1,58 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ package org.apache.hadoop.hdfs.server.datanode; -import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients; -import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; -import static org.apache.hadoop.ozone.OmUtils.getOmAddress; -import static org.apache.hadoop.ozone.OzoneConfigKeys.*; -import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS; -import static com.sun.jersey.api.core.ResourceConfig.FEATURE_TRACE; - -import java.io.Closeable; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.HashMap; -import java.util.Map; - import com.sun.jersey.api.container.ContainerFactory; import com.sun.jersey.api.core.ApplicationAdapter; - +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB; +import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; +import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.ipc.Client; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.om.protocolPB.OzoneManagerProtocolPB; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.web.ObjectStoreApplication; import org.apache.hadoop.ozone.web.handlers.ServiceFilter; +import org.apache.hadoop.ozone.web.interfaces.StorageHandler; import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer; -import org.apache.hadoop.hdds.scm.protocolPB - .ScmBlockLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB; +import org.apache.hadoop.ozone.web.storage.DistributedStorageHandler; +import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ipc.Client; -import org.apache.hadoop.ipc.ProtobufRpcEngine; -import org.apache.hadoop.ipc.RPC; -import org.apache.hadoop.net.NetUtils; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.protocolPB - .StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.hdds.scm.protocolPB.StorageContainerLocationProtocolPB; -import org.apache.hadoop.ozone.web.interfaces.StorageHandler; -import org.apache.hadoop.ozone.web.storage.DistributedStorageHandler; -import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler; -import org.apache.hadoop.security.UserGroupInformation; +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.Map; + +import static com.sun.jersey.api.core.ResourceConfig.FEATURE_TRACE; +import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS; +import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForBlockClients; +import static org.apache.hadoop.hdds.HddsUtils.getScmAddressForClients; +import static org.apache.hadoop.ozone.OmUtils.getOmAddress; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TRACE_ENABLED_DEFAULT; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_TRACE_ENABLED_KEY; /** * Implements object store handling within the DataNode process. This class is @@ -86,69 +80,49 @@ public final class ObjectStoreHandler implements Closeable { * @throws IOException if there is an I/O error */ public ObjectStoreHandler(Configuration conf) throws IOException { - String shType = conf.getTrimmed(OZONE_HANDLER_TYPE_KEY, - OZONE_HANDLER_TYPE_DEFAULT); - LOG.info("ObjectStoreHandler initializing with {}: {}", - OZONE_HANDLER_TYPE_KEY, shType); boolean ozoneTrace = conf.getBoolean(OZONE_TRACE_ENABLED_KEY, OZONE_TRACE_ENABLED_DEFAULT); // Initialize Jersey container for object store web application. - if (OzoneConsts.OZONE_HANDLER_DISTRIBUTED.equalsIgnoreCase(shType)) { - RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class, - ProtobufRpcEngine.class); - long scmVersion = - RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class); - - InetSocketAddress scmAddress = - getScmAddressForClients(conf); - this.storageContainerLocationClient = - new StorageContainerLocationProtocolClientSideTranslatorPB( - RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion, - scmAddress, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), - Client.getRpcTimeout(conf))); - - InetSocketAddress scmBlockAddress = - getScmAddressForBlockClients(conf); - this.scmBlockLocationClient = - new ScmBlockLocationProtocolClientSideTranslatorPB( - RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion, - scmBlockAddress, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), - Client.getRpcTimeout(conf))); - - RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class, - ProtobufRpcEngine.class); - long omVersion = - RPC.getProtocolVersion(OzoneManagerProtocolPB.class); - InetSocketAddress omAddress = getOmAddress(conf); - this.ozoneManagerClient = - new OzoneManagerProtocolClientSideTranslatorPB( - RPC.getProxy(OzoneManagerProtocolPB.class, omVersion, - omAddress, UserGroupInformation.getCurrentUser(), conf, - NetUtils.getDefaultSocketFactory(conf), - Client.getRpcTimeout(conf))); - - storageHandler = new DistributedStorageHandler( - new OzoneConfiguration(conf), - this.storageContainerLocationClient, - this.ozoneManagerClient); - } else { - if (OzoneConsts.OZONE_HANDLER_LOCAL.equalsIgnoreCase(shType)) { - storageHandler = new LocalStorageHandler(conf); - this.storageContainerLocationClient = null; - this.scmBlockLocationClient = null; - this.ozoneManagerClient = null; - } else { - throw new IllegalArgumentException( - String.format("Unrecognized value for %s: %s," - + " Allowed values are %s,%s", - OZONE_HANDLER_TYPE_KEY, shType, - OzoneConsts.OZONE_HANDLER_DISTRIBUTED, - OzoneConsts.OZONE_HANDLER_LOCAL)); - } - } + RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class, + ProtobufRpcEngine.class); + long scmVersion = + RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class); + + InetSocketAddress scmAddress = + getScmAddressForClients(conf); + this.storageContainerLocationClient = + new StorageContainerLocationProtocolClientSideTranslatorPB( + RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion, + scmAddress, UserGroupInformation.getCurrentUser(), conf, + NetUtils.getDefaultSocketFactory(conf), + Client.getRpcTimeout(conf))); + + InetSocketAddress scmBlockAddress = + getScmAddressForBlockClients(conf); + this.scmBlockLocationClient = + new ScmBlockLocationProtocolClientSideTranslatorPB( + RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion, + scmBlockAddress, UserGroupInformation.getCurrentUser(), conf, + NetUtils.getDefaultSocketFactory(conf), + Client.getRpcTimeout(conf))); + + RPC.setProtocolEngine(conf, OzoneManagerProtocolPB.class, + ProtobufRpcEngine.class); + long omVersion = + RPC.getProtocolVersion(OzoneManagerProtocolPB.class); + InetSocketAddress omAddress = getOmAddress(conf); + this.ozoneManagerClient = + new OzoneManagerProtocolClientSideTranslatorPB( + RPC.getProxy(OzoneManagerProtocolPB.class, omVersion, + omAddress, UserGroupInformation.getCurrentUser(), conf, + NetUtils.getDefaultSocketFactory(conf), + Client.getRpcTimeout(conf))); + + storageHandler = new DistributedStorageHandler( + new OzoneConfiguration(conf), + this.storageContainerLocationClient, + this.ozoneManagerClient); ApplicationAdapter aa = new ApplicationAdapter(new ObjectStoreApplication()); Map<String, Object> settingsMap = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/df21e1b1/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java index f86f247..b3c3391 100644 --- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java +++ b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java @@ -19,10 +19,11 @@ package org.apache.hadoop.ozone.web.handlers; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.web.interfaces.StorageHandler; -import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; /** * This class is responsible for providing a {@link StorageHandler} @@ -31,6 +32,9 @@ import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler; @InterfaceAudience.Private public final class StorageHandlerBuilder { + + private static final Logger LOG = + LoggerFactory.getLogger(StorageHandlerBuilder.class); private static final ThreadLocal<StorageHandler> STORAGE_HANDLER_THREAD_LOCAL = new ThreadLocal<>(); @@ -40,15 +44,15 @@ public final class StorageHandlerBuilder { * * @return StorageHandler from thread-local storage */ - public static StorageHandler getStorageHandler() { + public static StorageHandler getStorageHandler() throws IOException { StorageHandler storageHandler = STORAGE_HANDLER_THREAD_LOCAL.get(); if (storageHandler != null) { return storageHandler; } else { - // This only happens while using mvn jetty:run for testing. - Configuration conf = new OzoneConfiguration(); - return new LocalStorageHandler(conf); + LOG.error("No Storage Handler Configured."); + throw new IOException("Invalid Handler Configuration"); } + } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/df21e1b1/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java deleted file mode 100644 index 89158cb..0000000 --- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.web.localstorage; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ozone.client.io.LengthInputStream; -import org.apache.hadoop.ozone.client.rest.OzoneException; -import org.apache.hadoop.ozone.web.handlers.BucketArgs; -import org.apache.hadoop.ozone.web.handlers.KeyArgs; -import org.apache.hadoop.ozone.web.handlers.ListArgs; -import org.apache.hadoop.ozone.web.handlers.VolumeArgs; -import org.apache.hadoop.ozone.web.interfaces.StorageHandler; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.web.request.OzoneQuota; -import org.apache.hadoop.ozone.web.response.BucketInfo; -import org.apache.hadoop.ozone.web.response.KeyInfo; -import org.apache.hadoop.ozone.web.response.ListBuckets; -import org.apache.hadoop.ozone.web.response.ListKeys; -import org.apache.hadoop.ozone.web.response.ListVolumes; -import org.apache.hadoop.ozone.web.response.VolumeInfo; - -import java.io.IOException; -import java.io.OutputStream; - -/** - * PLEASE NOTE : This file is a dummy backend for test purposes and prototyping - * effort only. It does not handle any Object semantics correctly, neither does - * it take care of security. - */ -@InterfaceAudience.Private -public class LocalStorageHandler implements StorageHandler { - private final Configuration conf; - - /** - * Constructs LocalStorageHandler. - * - * @param conf ozone conf. - */ - public LocalStorageHandler(Configuration conf) { - this.conf = conf; - } - - /** - * Creates Storage Volume. - * - * @param args - volumeArgs - * @throws IOException - */ - @Override - public void createVolume(VolumeArgs args) throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.createVolume(args); - - } - - /** - * setVolumeOwner - sets the owner of the volume. - * - * @param args volumeArgs - * @throws IOException - */ - @Override - public void setVolumeOwner(VolumeArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.OWNER); - } - - /** - * Set Volume Quota Info. - * - * @param args - volumeArgs - * @param remove - true if the request is to remove the quota - * @throws IOException - */ - @Override - public void setVolumeQuota(VolumeArgs args, boolean remove) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - - if (remove) { - OzoneQuota quota = new OzoneQuota(); - args.setQuota(quota); - } - oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.QUOTA); - } - - /** - * Checks if a Volume exists and the user specified has access to the volume. - * - * @param volume - Volume Name - * @param acl - Ozone acl which needs to be compared for access - * @return - Boolean - True if the user can modify the volume. This is - * possible for owners of the volume and admin users - * @throws IOException - */ - @Override - public boolean checkVolumeAccess(String volume, OzoneAcl acl) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.checkVolumeAccess(volume, acl); - } - - /** - * Returns Info about the specified Volume. - * - * @param args - volumeArgs - * @return VolumeInfo - * @throws IOException - */ - @Override - public VolumeInfo getVolumeInfo(VolumeArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.getVolumeInfo(args); - } - - /** - * Deletes an Empty Volume. - * - * @param args - Volume Args - * @throws IOException - */ - @Override - public void deleteVolume(VolumeArgs args) throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.deleteVolume(args); - - } - - /** - * Returns the List of Volumes owned by the specific user. - * - * @param args - ListArgs - * @return - List of Volumes - * @throws IOException - */ - @Override - public ListVolumes listVolumes(ListArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.listVolumes(args); - } - - /** - * true if the bucket exists and user has read access to the bucket else - * throws Exception. - * - * @param args Bucket args structure - * @throws IOException - */ - @Override - public void checkBucketAccess(BucketArgs args) - throws IOException, OzoneException { - - } - - /** - * Creates a Bucket in specified Volume. - * - * @param args BucketArgs- BucketName, UserName and Acls - * @throws IOException - */ - @Override - public void createBucket(BucketArgs args) throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.createBucket(args); - } - - /** - * Adds or Removes ACLs from a Bucket. - * - * @param args - BucketArgs - * @throws IOException - */ - @Override - public void setBucketAcls(BucketArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.ACLS); - } - - /** - * Enables or disables Bucket Versioning. - * - * @param args - BucketArgs - * @throws IOException - */ - @Override - public void setBucketVersioning(BucketArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.VERSIONING); - - } - - /** - * Sets the Storage Class of a Bucket. - * - * @param args - BucketArgs - * @throws IOException - */ - @Override - public void setBucketStorageClass(BucketArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.STORAGETYPE); - - } - - /** - * Deletes a bucket if it is empty. - * - * @param args Bucket args structure - * @throws IOException - */ - @Override - public void deleteBucket(BucketArgs args) throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.deleteBucket(args); - } - - /** - * Returns all Buckets of a specified Volume. - * - * @param args --User Args - * @return ListAllBuckets - * @throws OzoneException - */ - @Override - public ListBuckets listBuckets(ListArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.listBuckets(args); - } - - /** - * Returns Bucket's Metadata as a String. - * - * @param args Bucket args structure - * @return Info about the bucket - * @throws IOException - */ - @Override - public BucketInfo getBucketInfo(BucketArgs args) - throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.getBucketInfo(args); - } - - /** - * Writes a key in an existing bucket. - * - * @param args KeyArgs - * @return InputStream - * @throws OzoneException - */ - @Override - public OutputStream newKeyWriter(KeyArgs args) throws IOException, - OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.createKey(args); - } - - /** - * Tells the file system that the object has been written out completely and - * it can do any house keeping operation that needs to be done. - * - * @param args Key Args - * @param stream - * @throws IOException - */ - @Override - public void commitKey(KeyArgs args, OutputStream stream) throws - IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.commitKey(args, stream); - - } - - /** - * Reads a key from an existing bucket. - * - * @param args KeyArgs - * @return LengthInputStream - * @throws IOException - */ - @Override - public LengthInputStream newKeyReader(KeyArgs args) throws IOException, - OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.newKeyReader(args); - } - - /** - * Deletes an existing key. - * - * @param args KeyArgs - * @throws OzoneException - */ - @Override - public void deleteKey(KeyArgs args) throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - oz.deleteKey(args); - } - - @Override - public void renameKey(KeyArgs args, String toKeyName) - throws IOException, OzoneException { - throw new UnsupportedOperationException("Not yet implemented"); - } - - /** - * Returns a list of Key. - * - * @param args KeyArgs - * @return BucketList - * @throws IOException - */ - @Override - public ListKeys listKeys(ListArgs args) throws IOException, OzoneException { - OzoneMetadataManager oz = - OzoneMetadataManager.getOzoneMetadataManager(conf); - return oz.listKeys(args); - - } - - /** - * Get information of the specified Key. - * - * @param args Key Args - * - * @return KeyInfo - * - * @throws IOException - * @throws OzoneException - */ - @Override - public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException { - OzoneMetadataManager oz = OzoneMetadataManager - .getOzoneMetadataManager(conf); - return oz.getKeyInfo(args); - } - - @Override - public void close() { - //No resource to close, do nothing. - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/df21e1b1/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java b/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java deleted file mode 100644 index 1fe9a18..0000000 --- a/hadoop-ozone/objectstore-service/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java +++ /dev/null @@ -1,1138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.web.localstorage; - -import com.google.common.base.Preconditions; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ozone.OzoneConfigKeys; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.client.io.LengthInputStream; -import org.apache.hadoop.ozone.web.exceptions.ErrorTable; -import org.apache.hadoop.ozone.client.rest.OzoneException; -import org.apache.hadoop.ozone.web.handlers.BucketArgs; -import org.apache.hadoop.ozone.web.handlers.KeyArgs; -import org.apache.hadoop.ozone.web.handlers.ListArgs; -import org.apache.hadoop.ozone.web.handlers.UserArgs; -import org.apache.hadoop.ozone.web.handlers.VolumeArgs; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.web.response.BucketInfo; -import org.apache.hadoop.ozone.web.response.KeyInfo; -import org.apache.hadoop.ozone.web.response.ListBuckets; -import org.apache.hadoop.ozone.web.response.ListKeys; -import org.apache.hadoop.ozone.web.response.ListVolumes; -import org.apache.hadoop.ozone.web.response.VolumeInfo; -import org.apache.hadoop.ozone.web.response.VolumeOwner; -import org.apache.hadoop.utils.MetadataStore; -import org.apache.hadoop.utils.MetadataStoreBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.charset.Charset; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.ListIterator; -import java.util.Locale; -import java.util.TimeZone; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -/** - * A stand alone Ozone implementation that allows us to run Ozone tests in local - * mode. This acts as the ozone backend when using MiniDFSCluster for testing. - */ -public final class OzoneMetadataManager { - - /* - OzoneMetadataManager manages volume/bucket/object metadata and - data. - - Metadata is maintained in 2 level DB files, UserDB and MetadataDB. - - UserDB contains a Name and a List. For example volumes owned by the user - bilbo, would be maintained in UserDB as {bilbo}->{shire, rings} - - This list part of mapping is context sensitive. That is, if you use {user - name} as the key, the list you get is a list of volumes. if you use - {user/volume} as the key the list you get is list of buckets. if you use - {user/volume/bucket} as key the list you get is the list of objects. - - All keys in the UserDB starts with the UserName. - - We also need to maintain a flat namespace for volumes. This is - maintained by the MetadataDB. MetadataDB contains the name of an - object(volume, bucket or key) and its associated metadata. - The keys in the Metadata DB are {volume}, {volume/bucket} or - {volume/bucket/key}. User name is absent, so we have a common root name - space for the volume. - - The value of part of metadataDB points to corresponding *Info structures. - {volume] -> volumeInfo - {volume/bucket} -> bucketInfo - {volume/bucket/key} -> keyInfo - - - Here are various work flows : - - CreateVolume -> Check if Volume exists in metadataDB, if not update UserDB - with a list of volumes and update metadataDB with VolumeInfo. - - DeleteVolume -> Check the Volume, and check the VolumeInfo->bucketCount. - if bucketCount == 0, delete volume from userDB->{List of volumes} and - metadataDB. - - Very similar work flows exist for CreateBucket and DeleteBucket. - - // Please note : These database operations are *not* transactional, - // which means that failure can lead to inconsistencies. - // Only way to recover is to reset to a clean state, or - // use rm -rf /tmp/ozone :) - - We have very simple locking policy. We have a ReaderWriter lock that is - taken for each action, this lock is aptly named "lock". - - All actions *must* be performed with a lock held, either a read - lock or a write lock. Violation of these locking policies can be harmful. - - - // // IMPORTANT : - // // This is a simulation layer, this is NOT how the real - // // OZONE functions. This is written to so that we can write - // // stand-alone tests for the protocol and client code. - -*/ - static final Logger LOG = LoggerFactory.getLogger(OzoneMetadataManager.class); - private static final String USER_DB = "/user.db"; - private static final String META_DB = "/metadata.db"; - private static OzoneMetadataManager bm = null; - private MetadataStore userDB; - private MetadataStore metadataDB; - private ReadWriteLock lock; - private Charset encoding = Charset.forName("UTF-8"); - private String storageRoot; - private static final String OBJECT_DIR = "/_objects/"; - - // This table keeps a pointer to objects whose operations - // are in progress but not yet committed to persistent store - private ConcurrentHashMap<OutputStream, String> inProgressObjects; - - /** - * Constructs OzoneMetadataManager. - */ - private OzoneMetadataManager(Configuration conf) throws IOException { - - lock = new ReentrantReadWriteLock(); - storageRoot = - conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT, - OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT); - - File file = new File(storageRoot + OBJECT_DIR); - - if (!file.exists() && !file.mkdirs()) { - LOG.error("Creation of Ozone root failed. " + file.toString()); - throw new IOException("Creation of Ozone root failed."); - } - - try { - userDB = MetadataStoreBuilder.newBuilder() - .setDbFile(new File(storageRoot + USER_DB)) - .setCreateIfMissing(true) - .build(); - metadataDB = MetadataStoreBuilder.newBuilder() - .setDbFile(new File(storageRoot + META_DB)) - .setCreateIfMissing(true) - .build(); - inProgressObjects = new ConcurrentHashMap<>(); - } catch (IOException ex) { - LOG.error("Cannot open db :" + ex.getMessage()); - throw ex; - } - } - - /** - * Gets Ozone Manager. - * - * @return OzoneMetadataManager - */ - public static synchronized OzoneMetadataManager - getOzoneMetadataManager(Configuration conf) throws IOException { - if (bm == null) { - bm = new OzoneMetadataManager(conf); - } - return bm; - } - - /** - * Creates a volume. - * - * @param args - VolumeArgs - * @throws OzoneException - */ - public void createVolume(VolumeArgs args) throws OzoneException { - lock.writeLock().lock(); - try { - SimpleDateFormat format = - new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US); - format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE)); - - byte[] volumeName = - metadataDB.get(args.getVolumeName().getBytes(encoding)); - - if (volumeName != null) { - LOG.debug("Volume {} already exists.", volumeName); - throw ErrorTable.newError(ErrorTable.VOLUME_ALREADY_EXISTS, args); - } - - VolumeInfo newVInfo = new VolumeInfo(args.getVolumeName(), format - .format(new Date(System.currentTimeMillis())), args.getAdminName()); - - newVInfo.setQuota(args.getQuota()); - VolumeOwner owner = new VolumeOwner(args.getUserName()); - newVInfo.setOwner(owner); - - ListVolumes volumeList; - byte[] userVolumes = userDB.get(args.getUserName().getBytes(encoding)); - if (userVolumes == null) { - volumeList = new ListVolumes(); - } else { - volumeList = ListVolumes.parse(new String(userVolumes, encoding)); - } - - volumeList.addVolume(newVInfo); - volumeList.sort(); - - // Please note : These database operations are *not* transactional, - // which means that failure can lead to inconsistencies. - // Only way to recover is to reset to a clean state, or - // use rm -rf /tmp/ozone :) - - - userDB.put(args.getUserName().getBytes(encoding), - volumeList.toDBString().getBytes(encoding)); - - metadataDB.put(args.getVolumeName().getBytes(encoding), - newVInfo.toDBString().getBytes(encoding)); - - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Updates the Volume properties like Owner Name and Quota. - * - * @param args - Volume Args - * @param property - Flag which tells us what property to upgrade - * @throws OzoneException - */ - public void setVolumeProperty(VolumeArgs args, VolumeProperty property) - throws OzoneException { - lock.writeLock().lock(); - try { - byte[] volumeInfo = - metadataDB.get(args.getVolumeName().getBytes(encoding)); - if (volumeInfo == null) { - throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args); - } - VolumeInfo info = VolumeInfo.parse(new String(volumeInfo, encoding)); - - byte[] userBytes = userDB.get(args.getResourceName().getBytes(encoding)); - ListVolumes volumeList; - if (userBytes == null) { - volumeList = new ListVolumes(); - } else { - volumeList = ListVolumes.parse(new String(userBytes, encoding)); - } - - switch (property) { - case OWNER: - // needs new owner, we delete the volume object from the - // old user's volume list - removeOldOwner(info); - VolumeOwner owner = new VolumeOwner(args.getUserName()); - // set the new owner - info.setOwner(owner); - break; - case QUOTA: - // if this is quota update we just remove the old object from the - // current users list and update the same object later. - volumeList.getVolumes().remove(info); - info.setQuota(args.getQuota()); - break; - default: - OzoneException ozEx = - ErrorTable.newError(ErrorTable.BAD_PROPERTY, args); - ozEx.setMessage("Volume property is not recognized"); - throw ozEx; - } - - volumeList.addVolume(info); - - metadataDB.put(args.getVolumeName().getBytes(encoding), - info.toDBString().getBytes(encoding)); - - // if this is an owner change this put will create a new owner or update - // the owner's volume list. - userDB.put(args.getResourceName().getBytes(encoding), - volumeList.toDBString().getBytes(encoding)); - - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Removes the old owner from the volume. - * - * @param info - VolumeInfo - * @throws IOException - */ - private void removeOldOwner(VolumeInfo info) throws IOException { - // We need to look the owner that we know is the current owner - byte[] volumeBytes = - userDB.get(info.getOwner().getName().getBytes(encoding)); - ListVolumes volumeList = - ListVolumes.parse(new String(volumeBytes, encoding)); - volumeList.getVolumes().remove(info); - - // Write the new list info to the old user data - userDB.put(info.getOwner().getName().getBytes(encoding), - volumeList.toDBString().getBytes(encoding)); - } - - /** - * Checks if you are the owner of a specific volume. - * - * @param volume - Volume Name whose access permissions needs to be checked - * @param acl - requested acls which needs to be checked for access - * @return - True if you are the owner, false otherwise - * @throws OzoneException - */ - public boolean checkVolumeAccess(String volume, OzoneAcl acl) - throws OzoneException { - lock.readLock().lock(); - try { - byte[] volumeInfo = - metadataDB.get(volume.getBytes(encoding)); - if (volumeInfo == null) { - throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, null); - } - - VolumeInfo info = VolumeInfo.parse(new String(volumeInfo, encoding)); - return info.getOwner().getName().equals(acl.getName()); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, null, ex); - } finally { - lock.readLock().unlock(); - } - } - - /** - * getVolumeInfo returns the Volume Info of a specific volume. - * - * @param args - Volume args - * @return VolumeInfo - * @throws OzoneException - */ - public VolumeInfo getVolumeInfo(VolumeArgs args) throws OzoneException { - lock.readLock().lock(); - try { - byte[] volumeInfo = - metadataDB.get(args.getVolumeName().getBytes(encoding)); - if (volumeInfo == null) { - throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args); - } - - return VolumeInfo.parse(new String(volumeInfo, encoding)); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Returns all the volumes owned by a specific user. - * - * @param args - User Args - * @return - ListVolumes - * @throws OzoneException - */ - public ListVolumes listVolumes(ListArgs args) throws OzoneException { - lock.readLock().lock(); - try { - if (args.isRootScan()) { - return listAllVolumes(args); - } - - UserArgs uArgs = (UserArgs) args.getArgs(); - byte[] volumeList = userDB.get(uArgs.getUserName().getBytes(encoding)); - if (volumeList == null) { - throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, uArgs); - } - - String prefix = args.getPrefix(); - int maxCount = args.getMaxKeys(); - String prevKey = args.getPrevKey(); - if (prevKey != null) { - // Format is username/volumeName, in local mode we don't use the - // user name since we have a userName DB. - String[] volName = args.getPrevKey().split("/"); - if (volName.length < 2) { - throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, uArgs); - } - prevKey = volName[1]; - } - return getFilteredVolumes(volumeList, prefix, prevKey, maxCount); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args.getArgs(), ex); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Returns a List of Volumes that meet the prefix, prevkey and maxCount - * constraints. - * - * @param volumeList - Byte Array of Volume Info. - * @param prefix - prefix string. - * @param prevKey - PrevKey - * @param maxCount - Maximum Count. - * @return ListVolumes. - * @throws IOException - */ - private ListVolumes getFilteredVolumes(byte[] volumeList, String prefix, - String prevKey, int maxCount) throws - IOException { - ListVolumes volumes = ListVolumes.parse(new String(volumeList, - encoding)); - int currentCount = 0; - ListIterator<VolumeInfo> iter = volumes.getVolumes().listIterator(); - ListVolumes filteredVolumes = new ListVolumes(); - while (currentCount < maxCount && iter.hasNext()) { - VolumeInfo vInfo = iter.next(); - if (isMatchingPrefix(prefix, vInfo) && isAfterKey(prevKey, vInfo)) { - filteredVolumes.addVolume(vInfo); - currentCount++; - } - } - return filteredVolumes; - } - - /** - * Returns all volumes in a cluster. - * - * @param args - ListArgs. - * @return ListVolumes. - * @throws OzoneException - */ - public ListVolumes listAllVolumes(ListArgs args) - throws OzoneException, IOException { - String prefix = args.getPrefix(); - final String prevKey; - int maxCount = args.getMaxKeys(); - String userName = null; - - if (args.getPrevKey() != null) { - // Format is username/volumeName - String[] volName = args.getPrevKey().split("/"); - if (volName.length < 2) { - throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs()); - } - - byte[] userNameBytes = userDB.get(volName[0].getBytes(encoding)); - userName = new String(userNameBytes, encoding); - prevKey = volName[1]; - } else { - userName = new String(userDB.peekAround(0, null).getKey(), encoding); - prevKey = null; - } - - if (userName == null || userName.isEmpty()) { - throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs()); - } - - ListVolumes returnSet = new ListVolumes(); - // we need to iterate through users until we get maxcount volumes - // or no more volumes are left. - userDB.iterate(null, (key, value) -> { - int currentSize = returnSet.getVolumes().size(); - if (currentSize < maxCount) { - String name = new String(key, encoding); - byte[] volumeList = userDB.get(name.getBytes(encoding)); - if (volumeList == null) { - throw new IOException( - ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs())); - } - returnSet.getVolumes().addAll( - getFilteredVolumes(volumeList, prefix, prevKey, - maxCount - currentSize).getVolumes()); - return true; - } else { - return false; - } - }); - - return returnSet; - } - - /** - * Checks if a name starts with a matching prefix. - * - * @param prefix - prefix string. - * @param vInfo - volume info. - * @return true or false. - */ - private boolean isMatchingPrefix(String prefix, VolumeInfo vInfo) { - if (prefix == null || prefix.isEmpty()) { - return true; - } - return vInfo.getVolumeName().startsWith(prefix); - } - - /** - * Checks if the key is after the prevKey. - * - * @param prevKey - String prevKey. - * @param vInfo - volume Info. - * @return - true or false. - */ - private boolean isAfterKey(String prevKey, VolumeInfo vInfo) { - if (prevKey == null || prevKey.isEmpty()) { - return true; - } - return prevKey.compareTo(vInfo.getVolumeName()) < 0; - } - - /** - * Deletes a volume if it exists and is empty. - * - * @param args - volume args - * @throws OzoneException - */ - public void deleteVolume(VolumeArgs args) throws OzoneException { - lock.writeLock().lock(); - try { - byte[] volumeName = - metadataDB.get(args.getVolumeName().getBytes(encoding)); - if (volumeName == null) { - throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args); - } - - VolumeInfo vInfo = VolumeInfo.parse(new String(volumeName, encoding)); - - // Only remove volumes if they are empty. - if (vInfo.getBucketCount() > 0) { - throw ErrorTable.newError(ErrorTable.VOLUME_NOT_EMPTY, args); - } - - ListVolumes volumeList; - String user = vInfo.getOwner().getName(); - byte[] userVolumes = userDB.get(user.getBytes(encoding)); - if (userVolumes == null) { - throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args); - } - - volumeList = ListVolumes.parse(new String(userVolumes, encoding)); - volumeList.getVolumes().remove(vInfo); - - metadataDB.delete(args.getVolumeName().getBytes(encoding)); - userDB.put(user.getBytes(encoding), - volumeList.toDBString().getBytes(encoding)); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Create a bucket if it does not exist. - * - * @param args - BucketArgs - * @throws OzoneException - */ - public void createBucket(BucketArgs args) throws OzoneException { - lock.writeLock().lock(); - try { - // check if volume exists, buckets cannot be created without volumes - byte[] volumeName = metadataDB.get(args.getVolumeName() - .getBytes(encoding)); - if (volumeName == null) { - throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args); - } - - // A resource name is volume/bucket -- That is the key in metadata table - byte[] bucketName = metadataDB.get(args.getResourceName() - .getBytes(encoding)); - if (bucketName != null) { - throw ErrorTable.newError(ErrorTable.BUCKET_ALREADY_EXISTS, args); - } - - BucketInfo bucketInfo = - new BucketInfo(args.getVolumeName(), args.getBucketName()); - - if (args.getRemoveAcls() != null) { - OzoneException ex = ErrorTable.newError(ErrorTable.MALFORMED_ACL, args); - ex.setMessage("Remove ACLs specified in bucket create. Please remove " - + "them and retry."); - throw ex; - } - - VolumeInfo volInfo = VolumeInfo.parse(new String(volumeName, encoding)); - volInfo.setBucketCount(volInfo.getBucketCount() + 1); - - bucketInfo.setAcls(args.getAddAcls()); - bucketInfo.setStorageType(args.getStorageType()); - bucketInfo.setVersioning(args.getVersioning()); - ListBuckets bucketList; - - // get bucket list from user/volume -> bucketList - byte[] volumeBuckets = userDB.get(args.getParentName() - .getBytes(encoding)); - if (volumeBuckets == null) { - bucketList = new ListBuckets(); - } else { - bucketList = ListBuckets.parse(new String(volumeBuckets, encoding)); - } - - bucketList.addBucket(bucketInfo); - bucketList.sort(); - - // Update Volume->bucketCount - userDB.put(args.getVolumeName().getBytes(encoding), - volInfo.toDBString().getBytes(encoding)); - - // Now update the userDB with user/volume -> bucketList - userDB.put(args.getParentName().getBytes(encoding), - bucketList.toDBString().getBytes(encoding)); - - // Update userDB with volume/bucket -> empty key list - userDB.put(args.getResourceName().getBytes(encoding), - new ListKeys().toDBString().getBytes(encoding)); - - // and update the metadataDB with volume/bucket->BucketInfo - metadataDB.put(args.getResourceName().getBytes(encoding), - bucketInfo.toDBString().getBytes(encoding)); - - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Updates the Bucket properties like ACls and Storagetype. - * - * @param args - Bucket Args - * @param property - Flag which tells us what property to upgrade - * @throws OzoneException - */ - public void setBucketProperty(BucketArgs args, BucketProperty property) - throws OzoneException { - - lock.writeLock().lock(); - try { - // volume/bucket-> bucketInfo - byte[] bucketInfo = metadataDB.get(args.getResourceName(). - getBytes(encoding)); - if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - - BucketInfo info = BucketInfo.parse(new String(bucketInfo, encoding)); - byte[] volumeBuckets = userDB.get(args.getParentName() - .getBytes(encoding)); - ListBuckets bucketList = ListBuckets.parse(new String(volumeBuckets, - encoding)); - bucketList.getBuckets().remove(info); - - switch (property) { - case ACLS: - processRemoveAcls(args, info); - processAddAcls(args, info); - break; - case STORAGETYPE: - info.setStorageType(args.getStorageType()); - break; - case VERSIONING: - info.setVersioning(args.getVersioning()); - break; - default: - OzoneException ozEx = - ErrorTable.newError(ErrorTable.BAD_PROPERTY, args); - ozEx.setMessage("Bucket property is not recognized."); - throw ozEx; - } - - bucketList.addBucket(info); - metadataDB.put(args.getResourceName().getBytes(encoding), - info.toDBString().getBytes(encoding)); - - userDB.put(args.getParentName().getBytes(encoding), - bucketList.toDBString().getBytes(encoding)); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Process Remove Acls and remove them from the bucket. - * - * @param args - BucketArgs - * @param info - BucketInfo - */ - private void processRemoveAcls(BucketArgs args, BucketInfo info) { - List<OzoneAcl> removeAcls = args.getRemoveAcls(); - if ((removeAcls == null) || (info.getAcls() == null)) { - return; - } - for (OzoneAcl racl : args.getRemoveAcls()) { - ListIterator<OzoneAcl> aclIter = info.getAcls().listIterator(); - while (aclIter.hasNext()) { - if (racl.equals(aclIter.next())) { - aclIter.remove(); - break; - } - } - } - } - - /** - * Process Add Acls and Add them to the bucket. - * - * @param args - BucketArgs - * @param info - BucketInfo - */ - private void processAddAcls(BucketArgs args, BucketInfo info) { - List<OzoneAcl> addAcls = args.getAddAcls(); - if ((addAcls == null)) { - return; - } - - if (info.getAcls() == null) { - info.setAcls(addAcls); - return; - } - - for (OzoneAcl newacl : addAcls) { - ListIterator<OzoneAcl> aclIter = info.getAcls().listIterator(); - while (aclIter.hasNext()) { - if (newacl.equals(aclIter.next())) { - continue; - } - } - info.getAcls().add(newacl); - } - } - - /** - * Deletes a given bucket. - * - * @param args - BucketArgs - * @throws OzoneException - */ - public void deleteBucket(BucketArgs args) throws OzoneException { - lock.writeLock().lock(); - try { - byte[] bucketInfo = metadataDB.get(args.getResourceName() - .getBytes(encoding)); - if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - - BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding)); - - // Only remove buckets if they are empty. - if (bInfo.getKeyCount() > 0) { - throw ErrorTable.newError(ErrorTable.BUCKET_NOT_EMPTY, args); - } - - byte[] bucketBytes = userDB.get(args.getParentName().getBytes(encoding)); - if (bucketBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - - ListBuckets bucketList = - ListBuckets.parse(new String(bucketBytes, encoding)); - bucketList.getBuckets().remove(bInfo); - - metadataDB.delete(args.getResourceName().getBytes(encoding)); - userDB.put(args.getParentName().getBytes(encoding), - bucketList.toDBString().getBytes(encoding)); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Returns the Bucket info for a given bucket. - * - * @param args - Bucket Args - * @return BucketInfo - Bucket Information - * @throws OzoneException - */ - public BucketInfo getBucketInfo(BucketArgs args) throws OzoneException { - lock.readLock().lock(); - try { - byte[] bucketBytes = metadataDB.get(args.getResourceName() - .getBytes(encoding)); - if (bucketBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - - return BucketInfo.parse(new String(bucketBytes, encoding)); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Returns a list of buckets for a given volume. - * - * @param args - volume args - * @return List of buckets - * @throws OzoneException - */ - public ListBuckets listBuckets(ListArgs args) throws OzoneException { - lock.readLock().lock(); - try { - Preconditions.checkState(args.getArgs() instanceof VolumeArgs); - VolumeArgs vArgs = (VolumeArgs) args.getArgs(); - String userVolKey = vArgs.getUserName() + "/" + vArgs.getVolumeName(); - - // TODO : Query using Prefix and PrevKey - byte[] bucketBytes = userDB.get(userVolKey.getBytes(encoding)); - if (bucketBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_VOLUME_NAME, - args.getArgs()); - } - return ListBuckets.parse(new String(bucketBytes, encoding)); - } catch (IOException ex) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args.getArgs(), ex); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Creates a key and returns a stream to which this key can be written to. - * - * @param args KeyArgs - * @return - A stream into which key can be written to. - * @throws OzoneException - */ - public OutputStream createKey(KeyArgs args) throws OzoneException { - lock.writeLock().lock(); - try { - String fileNameHash = DigestUtils.sha256Hex(args.getResourceName()); - - // Please don't try trillion objects unless the physical file system - // is capable of doing that in a single directory. - - String fullPath = storageRoot + OBJECT_DIR + fileNameHash; - File f = new File(fullPath); - - // In real ozone it would not be this way, a file will be overwritten - // only if the upload is successful. - if (f.exists()) { - LOG.debug("we are overwriting a file. This is by design."); - if (!f.delete()) { - LOG.error("Unable to delete the file: {}", fullPath); - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args); - } - } - - // f.createNewFile(); - FileOutputStream fsStream = new FileOutputStream(f); - inProgressObjects.put(fsStream, fullPath); - - return fsStream; - } catch (IOException e) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * commit keys moves an In progress object into the metadata store so that key - * is visible in the metadata operations from that point onwards. - * - * @param args Object args - * @throws OzoneException - */ - public void commitKey(KeyArgs args, OutputStream stream) - throws OzoneException { - SimpleDateFormat format = - new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US); - lock.writeLock().lock(); - - try { - byte[] bucketInfo = metadataDB.get(args.getParentName() - .getBytes(encoding)); - if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args); - } - BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding)); - bInfo.setKeyCount(bInfo.getKeyCount() + 1); - - String fileNameHash = inProgressObjects.get(stream); - inProgressObjects.remove(stream); - if (fileNameHash == null) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args); - } - - ListKeys keyList; - byte[] bucketListBytes = userDB.get(args.getParentName() - .getBytes(encoding)); - keyList = ListKeys.parse(new String(bucketListBytes, encoding)); - KeyInfo keyInfo; - - byte[] objectBytes = metadataDB.get(args.getResourceName() - .getBytes(encoding)); - - if (objectBytes != null) { - // we are overwriting an existing object. - // TODO : Emit info for Accounting - keyInfo = KeyInfo.parse(new String(objectBytes, encoding)); - keyList.getKeyList().remove(keyInfo); - } else { - keyInfo = new KeyInfo(); - } - - keyInfo.setCreatedOn(format.format(new Date(System.currentTimeMillis()))); - - // TODO : support version, we need to check if versioning - // is switched on the bucket and make appropriate calls. - keyInfo.setVersion(0); - - keyInfo.setDataFileName(fileNameHash); - keyInfo.setKeyName(args.getKeyName()); - keyInfo.setMd5hash(args.getHash()); - keyInfo.setSize(args.getSize()); - - keyList.getKeyList().add(keyInfo); - - // if the key exists, we overwrite happily :). since the - // earlier call - createObject - has overwritten the data. - - metadataDB.put(args.getResourceName().getBytes(encoding), - keyInfo.toDBString().getBytes(encoding)); - - metadataDB.put(args.getParentName().getBytes(encoding), - bInfo.toDBString().getBytes(encoding)); - - userDB.put(args.getParentName().getBytes(encoding), - keyList.toDBString().getBytes(encoding)); - - } catch (IOException e) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * deletes an key from a given bucket. - * - * @param args - ObjectArgs - * @throws OzoneException - */ - public void deleteKey(KeyArgs args) throws OzoneException { - lock.writeLock().lock(); - try { - byte[] bucketInfo = metadataDB.get(args.getParentName() - .getBytes(encoding)); - if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding)); - bInfo.setKeyCount(bInfo.getKeyCount() - 1); - - - byte[] bucketListBytes = userDB.get(args.getParentName() - .getBytes(encoding)); - if (bucketListBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - ListKeys keyList = ListKeys.parse(new String(bucketListBytes, encoding)); - - - byte[] objectBytes = metadataDB.get(args.getResourceName() - .getBytes(encoding)); - if (objectBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_KEY, args); - } - - KeyInfo oInfo = KeyInfo.parse(new String(objectBytes, encoding)); - keyList.getKeyList().remove(oInfo); - - String fileNameHash = DigestUtils.sha256Hex(args.getResourceName()); - - String fullPath = storageRoot + OBJECT_DIR + fileNameHash; - File f = new File(fullPath); - - if (f.exists()) { - if (!f.delete()) { - throw ErrorTable.newError(ErrorTable.KEY_OPERATION_CONFLICT, args); - } - } else { - throw ErrorTable.newError(ErrorTable.INVALID_KEY, args); - } - - - metadataDB.delete(args.getResourceName().getBytes(encoding)); - metadataDB.put(args.getParentName().getBytes(encoding), - bInfo.toDBString().getBytes(encoding)); - userDB.put(args.getParentName().getBytes(encoding), - keyList.toDBString().getBytes(encoding)); - } catch (IOException e) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e); - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Returns a Stream for the file. - * - * @param args - Object args - * @return Stream - * @throws IOException - * @throws OzoneException - */ - public LengthInputStream newKeyReader(KeyArgs args) - throws IOException, OzoneException { - lock.readLock().lock(); - try { - String fileNameHash = DigestUtils.sha256Hex(args.getResourceName()); - String fullPath = storageRoot + OBJECT_DIR + fileNameHash; - File f = new File(fullPath); - if (!f.exists()) { - throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args); - } - long size = f.length(); - - FileInputStream fileStream = new FileInputStream(f); - return new LengthInputStream(fileStream, size); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Returns keys in a bucket. - * - * @param args - * @return List of keys. - * @throws IOException - * @throws OzoneException - */ - public ListKeys listKeys(ListArgs args) throws IOException, OzoneException { - lock.readLock().lock(); - // TODO : Support Prefix and PrevKey lookup. - try { - Preconditions.checkState(args.getArgs() instanceof BucketArgs); - BucketArgs bArgs = (BucketArgs) args.getArgs(); - byte[] bucketInfo = metadataDB.get(bArgs.getResourceName() - .getBytes(encoding)); - if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs); - } - - byte[] bucketListBytes = userDB.get(bArgs.getResourceName() - .getBytes(encoding)); - if (bucketListBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs); - } - return ListKeys.parse(new String(bucketListBytes, encoding)); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Get the Key information for a given key. - * - * @param args - Key Args - * @return KeyInfo - Key Information - * @throws OzoneException - */ - public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException { - lock.readLock().lock(); - try { - byte[] bucketInfo = metadataDB - .get(args.getParentName().getBytes(encoding)); - if (bucketInfo == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - - byte[] bucketListBytes = userDB - .get(args.getParentName().getBytes(encoding)); - if (bucketListBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args); - } - - byte[] objectBytes = metadataDB - .get(args.getResourceName().getBytes(encoding)); - if (objectBytes == null) { - throw ErrorTable.newError(ErrorTable.INVALID_KEY, args); - } - - return KeyInfo.parse(new String(objectBytes, encoding)); - } catch (IOException e) { - throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e); - } finally { - lock.readLock().unlock(); - } - } - - /** - * This is used in updates to volume metadata. - */ - public enum VolumeProperty { - OWNER, QUOTA - } - - /** - * Bucket Properties. - */ - public enum BucketProperty { - ACLS, STORAGETYPE, VERSIONING - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/df21e1b1/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java index ad21f28..d3bc857 100644 --- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java +++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSInputStream.java @@ -57,8 +57,7 @@ public class TestOzoneFSInputStream { /** * Create a MiniDFSCluster for testing. * <p> - * Ozone is made active by setting OZONE_ENABLED = true and - * OZONE_HANDLER_TYPE_KEY = "distributed" + * Ozone is made active by setting OZONE_ENABLED = true * * @throws IOException */ @@ -90,9 +89,6 @@ public class TestOzoneFSInputStream { // Fetch the host and port for File System init DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0) .getDatanodeDetails(); - int port = datanodeDetails - .getPort(DatanodeDetails.Port.Name.REST).getValue(); - String host = datanodeDetails.getHostName(); // Set the fs.defaultFS and start the filesystem String uri = String.format("%s://%s.%s/", --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org