steveloughran commented on a change in pull request #1003: HADOOP-16384: Avoid inconsistencies between DDB and S3 URL: https://github.com/apache/hadoop/pull/1003#discussion_r302152332
########## File path: hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardDiagnostic.java ########## @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.s3guard; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.apache.hadoop.service.launcher.AbstractLaunchableService; +import org.apache.hadoop.service.launcher.LauncherExitCodes; +import org.apache.hadoop.service.launcher.ServiceLaunchException; + +import static org.apache.hadoop.service.launcher.LauncherExitCodes.EXIT_USAGE; + +/** + * Entry point for diagnostics operations. + */ +public class AbstractS3GuardDiagnostic extends AbstractLaunchableService { + + private S3AFileSystem filesystem; + + private DynamoDBMetadataStore store; + + private URI uri; + + private List<String> arguments; + + /** + * Constructor. + * @param name entry point name. + */ + public AbstractS3GuardDiagnostic(final String name) { + super(name); + } + + /** + * Constructor. If the store is set then that is the store for the operation, + * otherwise the filesystem's binding is used instead. + * @param name entry point name. + * @param filesystem filesystem + * @param store optional metastore. + * @param uri URI. Must be set if filesystem == null. + */ + public AbstractS3GuardDiagnostic( + final String name, + @Nullable final S3AFileSystem filesystem, + @Nullable final DynamoDBMetadataStore store, + @Nullable final URI uri) { + super(name); + this.store = store; + this.filesystem = filesystem; + if (store == null) { + require(filesystem != null, "No filesystem or URI"); + bindStore(filesystem); + } + if (uri == null) { + require(filesystem != null, "No filesystem or URI"); + setUri(filesystem.getUri()); + } else { + setUri(uri); + } + } + + /** + * Require a condition to hold, otherwise an exception is thrown. + * @param condition condition to be true + * @param error text on failure. + * @throws ServiceLaunchException if the condition is not met + */ + protected static void require(boolean condition, String error) { + if (!condition) { + throw failure(error); + } + } + + /** + * Generate a failure exception for throwing. + * @param message message + * @param ex optional nested exception. + * @return an exception to throw + */ + protected static ServiceLaunchException failure(String message, Throwable ex) { + return new ServiceLaunchException(LauncherExitCodes.EXIT_FAIL, message, ex); + } + + /** + * Generate a failure exception for throwing. + * @param message message + * @return an exception to throw + */ + protected static ServiceLaunchException failure(String message) { + return new ServiceLaunchException(LauncherExitCodes.EXIT_FAIL, message); + } + + @Override + public Configuration bindArgs(final Configuration config, + final List<String> args) + throws Exception { + this.arguments = args; + return super.bindArgs(config, args); + } + + /** + * Get the argument list. + * @return the argument list. + */ + protected List<String> getArguments() { + return arguments; + } + + /** + * Bind to the store from a CLI argument. + * @param fsURI filesystem URI + * @throws IOException failure + */ + protected void bindFromCLI(String fsURI) + throws IOException { + Configuration conf = getConfig(); + setUri(fsURI); + FileSystem fs = FileSystem.get(getUri(), conf); + require(fs instanceof S3AFileSystem, + "Not an S3A Filesystem: " + fsURI); + filesystem = (S3AFileSystem) fs; + bindStore(filesystem); + setUri(fs.getUri()); + } + + /** + * Binds the {@link #store} field to the metastore of + * the filesystem -which must have a DDB metastore. + * @param fs filesystem to bind the store to. + */ + private void bindStore(final S3AFileSystem fs) { + require(fs.hasMetadataStore(), + "Filesystem has no metadata store: " + fs.getUri()); + MetadataStore ms = fs.getMetadataStore(); + require(ms instanceof DynamoDBMetadataStore, Review comment: we're grabbing the DDB table and doing things with it -those scan commands in particular ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services --------------------------------------------------------------------- To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-issues-h...@hadoop.apache.org
