client blobstore interface documentation update
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6687bedb Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6687bedb Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6687bedb Branch: refs/heads/master Commit: 6687bedbe86c1b04cecf18e02e6e2359ce2c3ead Parents: 57240b4 Author: Sanket <schintap@untilservice-lm> Authored: Mon Nov 30 18:02:39 2015 -0600 Committer: Sanket <schintap@untilservice-lm> Committed: Mon Nov 30 18:02:39 2015 -0600 ---------------------------------------------------------------------- .../storm/hdfs/blobstore/HdfsBlobStore.java | 14 +++ .../hdfs/blobstore/HdfsClientBlobStore.java | 5 + .../storm/blobstore/ClientBlobStore.java | 126 ++++++++++++++++++- .../storm/blobstore/LocalFsBlobStore.java | 15 +++ .../storm/blobstore/NimbusBlobStore.java | 8 ++ 5 files changed, 166 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java ---------------------------------------------------------------------- diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java index 144ad71..c344bcf 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java @@ -54,6 +54,20 @@ import static backtype.storm.blobstore.BlobStoreAclHandler.WRITE; * Provides a HDFS file system backed blob store implementation. * Note that this provides an api for having HDFS be the backing store for the blobstore, * it is not a service/daemon. + * + * We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN + * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the blobs form the nimbus. + * + * The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER + * who has read, write or admin privileges in order to perform respective operations on the blob. + * + * For hdfs blob store + * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API. Here, unlike + * local blob store which stores the blobs locally, the nimbus talks to HDFS to upload the blobs. + * 2. The USER sets the ACLs, and the blob access is validated against these ACLs. + * 3. The SUPERVISOR interacts with nimbus thorugh HdfsClientBlobStore to download the blobs. Here, unlike local + * blob store the supervisor interacts with HDFS directly to download the blobs. The call to HdfsBlobStore is made as a "null" + * subject. The blobstore gets the hadoop user and validates permissions for the supervisor. */ public class HdfsBlobStore extends BlobStore { public static final Logger LOG = LoggerFactory.getLogger(HdfsBlobStore.class); http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java ---------------------------------------------------------------------- diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java index ec17dae..18c97da 100644 --- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java +++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsClientBlobStore.java @@ -35,6 +35,11 @@ import java.util.Map; /** * Client to access the HDFS blobStore. At this point, this is meant to only be used by the * supervisor. Don't trust who the client says they are so pass null for all Subjects. + * + * The HdfsBlobStore implementation takes care of the null Subjects. It assigns Subjects + * based on what hadoop says who the users are. These users must be configured accordingly + * in the SUPERVISOR_ADMINS for ACL validation and for the supervisors to download the blobs. + * This API is only used by the supervisor in order to talk directly to HDFS. */ public class HdfsClientBlobStore extends ClientBlobStore { private static final Logger LOG = LoggerFactory.getLogger(HdfsClientBlobStore.class); http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java b/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java index cc40aff..6408469 100644 --- a/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java +++ b/storm-core/src/jvm/backtype/storm/blobstore/ClientBlobStore.java @@ -28,22 +28,137 @@ import backtype.storm.utils.NimbusClient; import java.util.Iterator; import java.util.Map; +/** + * The ClientBlobStore has two concrete implementations + * 1. NimbusBlobStore + * 2. HdfsClientBlobStore + * + * Create, update, read and delete are some of the basic operations defined by this interface. + * Each operation is validated for permissions against an user. We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS + * configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN access whereas the SUPERVISOR_ADMINS are given READ + * access in order to read and download the blobs form the nimbus. + * + * The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER + * who has read, write or admin privileges in order to perform respective operations on the blob. + * + * For more detailed implementation + * @see backtype.storm.blobstore.NimbusBlobStore + * @see backtype.storm.blobstore.LocalFsBlobStore + * @see org.apache.storm.hdfs.blobstore.HdfsClientBlobStore + * @see org.apache.storm.hdfs.blobstore.HdfsBlobStore + */ public abstract class ClientBlobStore implements Shutdownable { protected Map conf; + /** + * Sets up the client API by parsing the configs. + * @param conf The storm conf containing the config details. + */ public abstract void prepare(Map conf); + + /** + * Client facing API to create a blob. + * @param key blob key name. + * @param meta contains ACL information. + * @return AtomicOutputStream returns an output stream into which data can be written. + * @throws AuthorizationException + * @throws KeyAlreadyExistsException + */ protected abstract AtomicOutputStream createBlobToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException; + + /** + * Client facing API to update a blob. + * @param key blob key name. + * @return AtomicOutputStream returns an output stream into which data can be written. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ public abstract AtomicOutputStream updateBlob(String key) throws AuthorizationException, KeyNotFoundException; + + /** + * Client facing API to read the metadata information. + * @param key blob key name. + * @return AtomicOutputStream returns an output stream into which data can be written. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ public abstract ReadableBlobMeta getBlobMeta(String key) throws AuthorizationException, KeyNotFoundException; + + /** + * Client facing API to set the metadata for a blob. + * @param key blob key name. + * @param meta contains ACL information. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ protected abstract void setBlobMetaToExtend(String key, SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException; + + /** + * Client facing API to delete a blob. + * @param key blob key name. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ public abstract void deleteBlob(String key) throws AuthorizationException, KeyNotFoundException; + + /** + * Client facing API to read a blob. + * @param key blob key name. + * @return an InputStream to read the metadata for a blob. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ public abstract InputStreamWithMeta getBlob(String key) throws AuthorizationException, KeyNotFoundException; + + /** + * @return Iterator for a list of keys currently present in the blob store. + */ public abstract Iterator<String> listKeys(); - public abstract int getBlobReplication(String Key) throws AuthorizationException, KeyNotFoundException; - public abstract int updateBlobReplication(String Key, int replication) throws AuthorizationException, KeyNotFoundException; + + /** + * Client facing API to read the replication of a blob. + * @param key blob key name. + * @return int indicates the replication factor of a blob. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ + public abstract int getBlobReplication(String key) throws AuthorizationException, KeyNotFoundException; + + /** + * Client facing API to update the replication of a blob. + * @param key blob key name. + * @param replication int indicates the replication factor a blob has to be set. + * @return int indicates the replication factor of a blob. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ + public abstract int updateBlobReplication(String key, int replication) throws AuthorizationException, KeyNotFoundException; + + /** + * Client facing API to set a nimbus client. + * @param conf storm conf + * @param client NimbusClient + * @return indicates where the client connection has been setup. + */ public abstract boolean setClient(Map conf, NimbusClient client); + + /** + * Creates state inside a zookeeper. + * Required for blobstore to write to zookeeper + * when Nimbus HA is turned on in order to maintain + * state consistency + * @param key + */ public abstract void createStateInZookeeper(String key); + /** + * Client facing API to create a blob. + * @param key blob key name. + * @param meta contains ACL information. + * @return AtomicOutputStream returns an output stream into which data can be written. + * @throws AuthorizationException + * @throws KeyAlreadyExistsException + */ public final AtomicOutputStream createBlob(String key, SettableBlobMeta meta) throws AuthorizationException, KeyAlreadyExistsException { if (meta !=null && meta.is_set_acl()) { BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl()); @@ -51,6 +166,13 @@ public abstract class ClientBlobStore implements Shutdownable { return createBlobToExtend(key, meta); } + /** + * Client facing API to set the metadata for a blob. + * @param key blob key name. + * @param meta contains ACL information. + * @throws AuthorizationException + * @throws KeyNotFoundException + */ public final void setBlobMeta(String key, SettableBlobMeta meta) throws AuthorizationException, KeyNotFoundException { if (meta !=null && meta.is_set_acl()) { BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl()); http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java b/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java index 0941b9a..ac7a4bd 100644 --- a/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java +++ b/storm-core/src/jvm/backtype/storm/blobstore/LocalFsBlobStore.java @@ -48,6 +48,21 @@ import static backtype.storm.blobstore.BlobStoreAclHandler.WRITE; /** * Provides a local file system backed blob store implementation for Nimbus. + * + * For a local blob store the user and the supervisor use NimbusBlobStore Client API in order to talk to nimbus through thrift. + * The authentication and authorization here is based on the subject. + * We currently have NIMBUS_ADMINS and SUPERVISOR_ADMINS configuration. NIMBUS_ADMINS are given READ, WRITE and ADMIN + * access whereas the SUPERVISOR_ADMINS are given READ access in order to read and download the blobs form the nimbus. + * + * The ACLs for the blob store are validated against whether the subject is a NIMBUS_ADMIN, SUPERVISOR_ADMIN or USER + * who has read, write or admin privileges in order to perform respective operations on the blob. + * + * For local blob store + * 1. The USER interacts with nimbus to upload and access blobs through NimbusBlobStore Client API. + * 2. The USER sets the ACLs, and the blob access is validated against these ACLs. + * 3. The SUPERVISOR interacts with nimbus through the NimbusBlobStore Client API to download the blobs. + * The supervisors principal should match the set of users configured into SUPERVISOR_ADMINS. + * Here, the PrincipalToLocalPlugin takes care of mapping the principal to user name before the ACL validation. */ public class LocalFsBlobStore extends BlobStore { public static final Logger LOG = LoggerFactory.getLogger(LocalFsBlobStore.class); http://git-wip-us.apache.org/repos/asf/storm/blob/6687bedb/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java b/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java index bf084bb..334e6bb 100644 --- a/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java +++ b/storm-core/src/jvm/backtype/storm/blobstore/NimbusBlobStore.java @@ -37,6 +37,14 @@ import java.util.Iterator; import java.util.Map; import java.util.NoSuchElementException; +/** + * NimbusBlobStore is a USER facing client API to perform + * basic operations such as create, update, delete and read + * for local and hdfs blob store. + * + * For local blob store it is also the client facing API for + * supervisor in order to download blobs from nimbus. + */ public class NimbusBlobStore extends ClientBlobStore { private static final Logger LOG = LoggerFactory.getLogger(NimbusBlobStore.class);
