REVERTED: DRILL-5089 Dynamically load schema of storage plugin only when needed for every query
Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/450e6709 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/450e6709 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/450e6709 Branch: refs/heads/master Commit: 450e67094eb6e9a6484d7f86c49b51c77a08d7b2 Parents: ef0fafe Author: Roman Kulyk <rom.ku...@gmail.com> Authored: Thu Nov 30 16:19:12 2017 +0000 Committer: Volodymyr Vysotskyi <vvo...@gmail.com> Committed: Tue Jan 16 12:10:13 2018 +0200 ---------------------------------------------------------------------- .../apache/drill/exec/ops/FragmentContext.java | 9 +- .../org/apache/drill/exec/ops/QueryContext.java | 11 +- .../exec/planner/sql/DynamicRootSchema.java | 142 ------------------- .../drill/exec/planner/sql/DynamicSchema.java | 58 -------- .../drill/exec/planner/sql/SqlConverter.java | 9 +- .../drill/exec/store/SchemaTreeProvider.java | 31 +--- .../exec/store/StoragePluginRegistryImpl.java | 3 - .../exec/store/dfs/FileSystemSchemaFactory.java | 23 +-- .../exec/store/dfs/WorkspaceSchemaFactory.java | 87 ++++-------- .../store/ischema/InfoSchemaBatchCreator.java | 2 +- .../exec/store/mock/MockBreakageStorage.java | 47 ------ .../exec/store/mock/MockStorageEngine.java | 7 +- .../exec/work/metadata/MetadataProvider.java | 2 +- .../drill/exec/physical/impl/TestSchema.java | 87 ------------ .../drill/test/ClusterFixtureBuilder.java | 4 - .../drill/test/ClusterMockStorageFixture.java | 51 ------- 16 files changed, 46 insertions(+), 527 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java index 210d0d4..736d550 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/FragmentContext.java @@ -230,12 +230,7 @@ public class FragmentContext extends BaseFragmentContext implements AutoCloseabl return context; } - /** - * This method is only used to construt InfoSchemaReader, it is for the reader to get full schema, so here we - * are going to return a fully initialized schema tree. - * @return root schema's plus - */ - public SchemaPlus getFullRootSchema() { + public SchemaPlus getRootSchema() { if (queryContext == null) { fail(new UnsupportedOperationException("Schema tree can only be created in root fragment. " + "This is a non-root fragment.")); @@ -253,7 +248,7 @@ public class FragmentContext extends BaseFragmentContext implements AutoCloseabl .setIgnoreAuthErrors(isImpersonationEnabled) .build(); - return queryContext.getFullRootSchema(schemaConfig); + return queryContext.getRootSchema(schemaConfig); } /** http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java index eb32bc6..8dbddbf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ops/QueryContext.java @@ -163,23 +163,14 @@ public class QueryContext implements AutoCloseable, OptimizerRulesContext, Schem } /** - * Create and return a SchemaTree with given <i>schemaConfig</i> but some schemas (from storage plugins) - * could be initialized later. + * Create and return a SchemaTree with given <i>schemaConfig</i>. * @param schemaConfig * @return */ public SchemaPlus getRootSchema(SchemaConfig schemaConfig) { return schemaTreeProvider.createRootSchema(schemaConfig); } - /** - * Create and return a fully initialized SchemaTree with given <i>schemaConfig</i>. - * @param schemaConfig - * @return - */ - public SchemaPlus getFullRootSchema(SchemaConfig schemaConfig) { - return schemaTreeProvider.createFullRootSchema(schemaConfig); - } /** * Get the user name of the user who issued the query that is managed by this QueryContext. * @return http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicRootSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicRootSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicRootSchema.java deleted file mode 100644 index df504b7..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicRootSchema.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.sql; - -import com.google.common.collect.ImmutableSortedSet; -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import org.apache.calcite.DataContext; -import org.apache.calcite.jdbc.CalciteRootSchema; -import org.apache.calcite.jdbc.CalciteSchema; - -import org.apache.calcite.linq4j.tree.Expression; -import org.apache.calcite.linq4j.tree.Expressions; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.calcite.schema.impl.AbstractSchema; -import org.apache.calcite.util.BuiltInMethod; -import org.apache.calcite.util.Compatible; -import org.apache.drill.common.exceptions.ExecutionSetupException; -import org.apache.drill.exec.store.SchemaConfig; -import org.apache.drill.exec.store.StoragePlugin; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.SubSchemaWrapper; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.NavigableSet; -import java.util.Set; - -/** - * This class is to allow us loading schemas from storage plugins later when {@link #getSubSchema(String, boolean)} - * is called. - */ -public class DynamicRootSchema extends DynamicSchema - implements CalciteRootSchema { - private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DynamicRootSchema.class); - - protected SchemaConfig schemaConfig; - protected StoragePluginRegistry storages; - - public StoragePluginRegistry getSchemaFactories() { - return storages; - } - - /** Creates a root schema. */ - DynamicRootSchema(StoragePluginRegistry storages, SchemaConfig schemaConfig) { - super(null, new RootSchema(), ""); - this.schemaConfig = schemaConfig; - this.storages = storages; - } - - @Override - public CalciteSchema getSubSchema(String schemaName, boolean caseSensitive) { - CalciteSchema retSchema = getSubSchemaMap().get(schemaName); - if (retSchema != null) { - return retSchema; - } - - loadSchemaFactory(schemaName, caseSensitive); - retSchema = getSubSchemaMap().get(schemaName); - return retSchema; - } - - @Override - public NavigableSet<String> getTableNames() { - return Compatible.INSTANCE.navigableSet(ImmutableSortedSet.<String>of()); - } - - /** - * load schema factory(storage plugin) for schemaName - * @param schemaName - * @param caseSensitive - */ - public void loadSchemaFactory(String schemaName, boolean caseSensitive) { - try { - SchemaPlus thisPlus = this.plus(); - StoragePlugin plugin = getSchemaFactories().getPlugin(schemaName); - if (plugin != null) { - plugin.registerSchemas(schemaConfig, thisPlus); - return; - } - - // Could not find the plugin of schemaName. The schemaName could be `dfs.tmp`, a 2nd level schema under 'dfs' - String[] paths = schemaName.split("\\."); - if (paths.length == 2) { - plugin = getSchemaFactories().getPlugin(paths[0]); - if (plugin == null) { - return; - } - - // Found the storage plugin for first part(e.g. 'dfs') of schemaName (e.g. 'dfs.tmp') - // register schema for this storage plugin to 'this'. - plugin.registerSchemas(schemaConfig, thisPlus); - - // Load second level schemas for this storage plugin - final SchemaPlus firstlevelSchema = thisPlus.getSubSchema(paths[0]); - final List<SchemaPlus> secondLevelSchemas = Lists.newArrayList(); - for (String secondLevelSchemaName : firstlevelSchema.getSubSchemaNames()) { - secondLevelSchemas.add(firstlevelSchema.getSubSchema(secondLevelSchemaName)); - } - - for (SchemaPlus schema : secondLevelSchemas) { - org.apache.drill.exec.store.AbstractSchema drillSchema; - try { - drillSchema = schema.unwrap(org.apache.drill.exec.store.AbstractSchema.class); - } catch (ClassCastException e) { - throw new RuntimeException(String.format("Schema '%s' is not expected under root schema", schema.getName())); - } - SubSchemaWrapper wrapper = new SubSchemaWrapper(drillSchema); - thisPlus.add(wrapper.getName(), wrapper); - } - } - } catch(ExecutionSetupException | IOException ex) { - logger.warn("Failed to load schema for \"" + schemaName + "\"!", ex); - } - } - - static class RootSchema extends AbstractSchema { - @Override public Expression getExpression(SchemaPlus parentSchema, - String name) { - return Expressions.call( - DataContext.ROOT, - BuiltInMethod.DATA_CONTEXT_GET_ROOT_SCHEMA.method); - } - } -} - http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicSchema.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicSchema.java deleted file mode 100644 index 7211f0d..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DynamicSchema.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.planner.sql; - -import org.apache.calcite.jdbc.CalciteSchema; -import org.apache.calcite.jdbc.SimpleCalciteSchema; -import org.apache.calcite.schema.Schema; -import org.apache.calcite.schema.SchemaPlus; -import org.apache.drill.exec.store.SchemaConfig; -import org.apache.drill.exec.store.StoragePluginRegistry; - - -/** - * Unlike SimpleCalciteSchema, DynamicSchema could have an empty or partial schemaMap, but it could maintain a map of - * name->SchemaFactory, and only register schema when the corresponsdent name is requested. - */ -public class DynamicSchema extends SimpleCalciteSchema { - - public DynamicSchema(CalciteSchema parent, Schema schema, String name) { - super(parent, schema, name); - } - - @Override - public CalciteSchema getSubSchema(String schemaName, boolean caseSensitive) { - Schema s = schema.getSubSchema(schemaName); - if (s != null) { - return new DynamicSchema(this, s, schemaName); - } - CalciteSchema ret = getSubSchemaMap().get(schemaName); - return ret; - } - - @Override - public SchemaPlus plus() { - return super.plus(); - } - - public static SchemaPlus createRootSchema(StoragePluginRegistry storages, SchemaConfig schemaConfig) { - DynamicRootSchema rootSchema = new DynamicRootSchema(storages, schemaConfig); - return rootSchema.plus(); - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java index 11b90e2..dbe16bd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java @@ -24,6 +24,7 @@ import java.util.Set; import com.google.common.base.Strings; import org.apache.calcite.adapter.java.JavaTypeFactory; import org.apache.calcite.jdbc.CalciteSchema; +import org.apache.calcite.jdbc.CalciteSchemaImpl; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelOptCluster; @@ -116,9 +117,9 @@ public class SqlConverter { this.session = context.getSession(); this.drillConfig = context.getConfig(); this.catalog = new DrillCalciteCatalogReader( - rootSchema, + this.rootSchema, parserConfig.caseSensitive(), - DynamicSchema.from(defaultSchema).path(null), + CalciteSchemaImpl.from(defaultSchema).path(null), typeFactory, drillConfig, session); @@ -296,7 +297,7 @@ public class SqlConverter { @Override public RelNode expandView(RelDataType rowType, String queryString, SchemaPlus rootSchema, List<String> schemaPath) { final DrillCalciteCatalogReader catalogReader = new DrillCalciteCatalogReader( - rootSchema, + rootSchema, // new root schema parserConfig.caseSensitive(), schemaPath, typeFactory, @@ -445,7 +446,7 @@ public class SqlConverter { JavaTypeFactory typeFactory, DrillConfig drillConfig, UserSession session) { - super(DynamicSchema.from(rootSchema), caseSensitive, defaultSchema, typeFactory); + super(CalciteSchemaImpl.from(rootSchema), caseSensitive, defaultSchema, typeFactory); this.drillConfig = drillConfig; this.session = session; this.allowTemporaryTables = true; http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaTreeProvider.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaTreeProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaTreeProvider.java index e1a1ede..23441bd 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaTreeProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/SchemaTreeProvider.java @@ -20,12 +20,12 @@ package org.apache.drill.exec.store; import java.io.IOException; import java.util.List; +import org.apache.calcite.jdbc.SimpleCalciteSchema; import org.apache.calcite.schema.SchemaPlus; import org.apache.drill.common.AutoCloseables; import org.apache.drill.common.exceptions.UserException; import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.ops.ViewExpansionContext; -import org.apache.drill.exec.planner.sql.DynamicSchema; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.server.options.OptionManager; import org.apache.drill.exec.server.options.OptionValue; @@ -105,36 +105,12 @@ public class SchemaTreeProvider implements AutoCloseable { * @return */ public SchemaPlus createRootSchema(SchemaConfig schemaConfig) { - final SchemaPlus rootSchema = DynamicSchema.createRootSchema(dContext.getStorage(), schemaConfig); - schemaTreesToClose.add(rootSchema); - return rootSchema; - } - - /** - * Return full root schema with schema owner as the given user. - * - * @param userName Name of the user who is accessing the storage sources. - * @param provider {@link SchemaConfigInfoProvider} instance - * @return Root of the schema tree. - */ - public SchemaPlus createFullRootSchema(final String userName, final SchemaConfigInfoProvider provider) { - final String schemaUser = isImpersonationEnabled ? userName : ImpersonationUtil.getProcessUserName(); - final SchemaConfig schemaConfig = SchemaConfig.newBuilder(schemaUser, provider).build(); - return createFullRootSchema(schemaConfig); - } - /** - * Create and return a Full SchemaTree with given <i>schemaConfig</i>. - * @param schemaConfig - * @return - */ - public SchemaPlus createFullRootSchema(SchemaConfig schemaConfig) { try { - final SchemaPlus rootSchema = DynamicSchema.createRootSchema(dContext.getStorage(), schemaConfig); + final SchemaPlus rootSchema = SimpleCalciteSchema.createRootSchema(false); dContext.getSchemaFactory().registerSchemas(schemaConfig, rootSchema); schemaTreesToClose.add(rootSchema); return rootSchema; - } - catch(IOException e) { + } catch(IOException e) { // We can't proceed further without a schema, throw a runtime exception. // Improve the error message for client side. @@ -148,7 +124,6 @@ public class SchemaTreeProvider implements AutoCloseable { .addContext(contextString) .build(logger); } - } @Override http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java index 2b4d648..3fb1c3a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/StoragePluginRegistryImpl.java @@ -46,7 +46,6 @@ import org.apache.drill.exec.ExecConstants; import org.apache.drill.exec.exception.DrillbitStartupException; import org.apache.drill.exec.exception.StoreException; import org.apache.drill.exec.planner.logical.StoragePlugins; -import org.apache.drill.exec.planner.sql.DynamicSchema; import org.apache.drill.exec.server.DrillbitContext; import org.apache.drill.exec.store.dfs.FileSystemPlugin; import org.apache.drill.exec.store.dfs.FormatPlugin; @@ -495,6 +494,4 @@ public class StoragePluginRegistryImpl implements StoragePluginRegistry { return availablePlugins; } - - } http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java index 6d88d04..5d99377 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSystemSchemaFactory.java @@ -27,7 +27,6 @@ import org.apache.calcite.schema.Function; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; -import org.apache.drill.exec.store.StoragePlugin; import org.apache.drill.exec.store.StorageStrategy; import org.apache.drill.exec.planner.logical.CreateTableEntry; import org.apache.drill.exec.store.AbstractSchema; @@ -39,9 +38,7 @@ import org.apache.drill.exec.store.dfs.WorkspaceSchemaFactory.WorkspaceSchema; import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import org.apache.drill.exec.util.DrillFileSystemUtil; -import org.apache.drill.exec.util.ImpersonationUtil; import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -52,23 +49,11 @@ public class FileSystemSchemaFactory implements SchemaFactory{ public static final String DEFAULT_WS_NAME = "default"; - public static final String LOCAL_FS_SCHEME = "file"; - private List<WorkspaceSchemaFactory> factories; private String schemaName; - protected FileSystemPlugin plugin; public FileSystemSchemaFactory(String schemaName, List<WorkspaceSchemaFactory> factories) { - // when the correspondent FileSystemPlugin is not passed in, we dig into ANY workspace factory to get it. - if (factories.size() > 0) { - this.plugin = factories.get(0).getPlugin(); - } - this.schemaName = schemaName; - this.factories = factories; - } - - public FileSystemSchemaFactory(FileSystemPlugin plugin, String schemaName, List<WorkspaceSchemaFactory> factories) { - this.plugin = plugin; + super(); this.schemaName = schemaName; this.factories = factories; } @@ -88,10 +73,10 @@ public class FileSystemSchemaFactory implements SchemaFactory{ public FileSystemSchema(String name, SchemaConfig schemaConfig) throws IOException { super(ImmutableList.<String>of(), name); - final DrillFileSystem fs = ImpersonationUtil.createFileSystem(schemaConfig.getUserName(), plugin.getFsConf()); for(WorkspaceSchemaFactory f : factories){ - WorkspaceSchema s = f.createSchema(getSchemaPath(), schemaConfig, fs); - if (s != null) { + if (f.accessible(schemaConfig.getUserName())) { + @SuppressWarnings("resource") + WorkspaceSchema s = f.createSchema(getSchemaPath(), schemaConfig); schemaMap.put(s.getName(), s); } } http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java index 3934958..bbf013d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/WorkspaceSchemaFactory.java @@ -42,7 +42,6 @@ import org.apache.calcite.schema.FunctionParameter; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.TableMacro; import org.apache.calcite.schema.TranslatableTable; -import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.common.config.LogicalPlanPersistence; import org.apache.drill.common.exceptions.ExecutionSetupException; @@ -71,7 +70,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.AccessControlException; @@ -152,30 +150,14 @@ public class WorkspaceSchemaFactory { * @return True if the user has access. False otherwise. */ public boolean accessible(final String userName) throws IOException { - final DrillFileSystem fs = ImpersonationUtil.createFileSystem(userName, fsConf); - return accessible(fs); - } - - /** - * Checks whether a FileSystem object has the permission to list/read workspace directory - * @param fs a DrillFileSystem object that was created with certain user privilege - * @return True if the user has access. False otherwise. - * @throws IOException - */ - public boolean accessible(DrillFileSystem fs) throws IOException { + final FileSystem fs = ImpersonationUtil.createFileSystem(userName, fsConf); try { - /** - * For Windows local file system, fs.access ends up using DeprecatedRawLocalFileStatus which has - * TrustedInstaller as owner, and a member of Administrators group could not satisfy the permission. - * In this case, we will still use method listStatus. - * In other cases, we use access method since it is cheaper. - */ - if (SystemUtils.IS_OS_WINDOWS && fs.getUri().getScheme().equalsIgnoreCase(FileSystemSchemaFactory.LOCAL_FS_SCHEME)) { + // We have to rely on the listStatus as a FileSystem can have complicated controls such as regular unix style + // permissions, Access Control Lists (ACLs) or Access Control Expressions (ACE). Hadoop 2.7 version of FileSystem + // has a limited private API (FileSystem.access) to check the permissions directly + // (see https://issues.apache.org/jira/browse/HDFS-6570). Drill currently relies on Hadoop 2.5.0 version of + // FileClient. TODO: Update this when DRILL-3749 is fixed. fs.listStatus(wsPath); - } - else { - fs.access(wsPath, FsAction.READ); - } } catch (final UnsupportedOperationException e) { logger.trace("The filesystem for this workspace does not support this operation.", e); } catch (final FileNotFoundException | AccessControlException e) { @@ -189,19 +171,8 @@ public class WorkspaceSchemaFactory { return DotDrillType.VIEW.getPath(config.getLocation(), name); } - public WorkspaceSchema createSchema(List<String> parentSchemaPath, SchemaConfig schemaConfig, DrillFileSystem fs) throws IOException { - if (!accessible(fs)) { - return null; - } - return new WorkspaceSchema(parentSchemaPath, schemaName, schemaConfig, fs); - } - - public String getSchemaName() { - return schemaName; - } - - public FileSystemPlugin getPlugin() { - return plugin; + public WorkspaceSchema createSchema(List<String> parentSchemaPath, SchemaConfig schemaConfig) throws IOException { + return new WorkspaceSchema(parentSchemaPath, schemaName, schemaConfig); } /** @@ -409,12 +380,12 @@ public class WorkspaceSchemaFactory { public class WorkspaceSchema extends AbstractSchema implements ExpandingConcurrentMap.MapValueFactory<TableInstance, DrillTable> { private final ExpandingConcurrentMap<TableInstance, DrillTable> tables = new ExpandingConcurrentMap<>(this); private final SchemaConfig schemaConfig; - private DrillFileSystem fs; + private final DrillFileSystem fs; - public WorkspaceSchema(List<String> parentSchemaPath, String wsName, SchemaConfig schemaConfig, DrillFileSystem fs) throws IOException { + public WorkspaceSchema(List<String> parentSchemaPath, String wsName, SchemaConfig schemaConfig) throws IOException { super(parentSchemaPath, wsName); this.schemaConfig = schemaConfig; - this.fs = fs; + this.fs = ImpersonationUtil.createFileSystem(schemaConfig.getUserName(), fsConf); } DrillTable getDrillTable(TableInstance key) { @@ -424,10 +395,10 @@ public class WorkspaceSchemaFactory { @Override public boolean createView(View view) throws IOException { Path viewPath = getViewPath(view.getName()); - boolean replaced = getFS().exists(viewPath); + boolean replaced = fs.exists(viewPath); final FsPermission viewPerms = new FsPermission(schemaConfig.getOption(ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY).string_val); - try (OutputStream stream = DrillFileSystem.create(getFS(), viewPath, viewPerms)) { + try (OutputStream stream = DrillFileSystem.create(fs, viewPath, viewPerms)) { mapper.writeValue(stream, view); } return replaced; @@ -450,7 +421,7 @@ public class WorkspaceSchemaFactory { @Override public void dropView(String viewName) throws IOException { - getFS().delete(getViewPath(viewName), false); + fs.delete(getViewPath(viewName), false); } private Set<String> getViews() { @@ -458,7 +429,7 @@ public class WorkspaceSchemaFactory { // Look for files with ".view.drill" extension. List<DotDrillFile> files; try { - files = DotDrillUtil.getDotDrills(getFS(), new Path(config.getLocation()), DotDrillType.VIEW); + files = DotDrillUtil.getDotDrills(fs, new Path(config.getLocation()), DotDrillType.VIEW); for (DotDrillFile f : files) { viewSet.add(f.getBaseName()); } @@ -527,7 +498,7 @@ public class WorkspaceSchemaFactory { List<DotDrillFile> files = Collections.emptyList(); try { try { - files = DotDrillUtil.getDotDrills(getFS(), new Path(config.getLocation()), tableName, DotDrillType.VIEW); + files = DotDrillUtil.getDotDrills(fs, new Path(config.getLocation()), tableName, DotDrillType.VIEW); } catch (AccessControlException e) { if (!schemaConfig.getIgnoreAuthErrors()) { logger.debug(e.getMessage()); @@ -599,19 +570,18 @@ public class WorkspaceSchemaFactory { } private DrillTable isReadable(FormatMatcher m, FileSelection fileSelection) throws IOException { - return m.isReadable(getFS(), fileSelection, plugin, storageEngineName, schemaConfig.getUserName()); + return m.isReadable(fs, fileSelection, plugin, storageEngineName, schemaConfig.getUserName()); } @Override public DrillTable create(TableInstance key) { try { - final FileSelection fileSelection = FileSelection - .create(getFS(), config.getLocation(), key.sig.name, config.allowAccessOutsideWorkspace()); + final FileSelection fileSelection = FileSelection.create(fs, config.getLocation(), key.sig.name, config.allowAccessOutsideWorkspace()); if (fileSelection == null) { return null; } - final boolean hasDirectories = fileSelection.containsDirectories(getFS()); + final boolean hasDirectories = fileSelection.containsDirectories(fs); if (key.sig.params.size() > 0) { FormatPluginConfig fconfig = optionExtractor.createConfigForTable(key); return new DynamicDrillTable( @@ -621,7 +591,7 @@ public class WorkspaceSchemaFactory { if (hasDirectories) { for (final FormatMatcher matcher : dirMatchers) { try { - DrillTable table = matcher.isReadable(getFS(), fileSelection, plugin, storageEngineName, schemaConfig.getUserName()); + DrillTable table = matcher.isReadable(fs, fileSelection, plugin, storageEngineName, schemaConfig.getUserName()); if (table != null) { return table; } @@ -631,13 +601,13 @@ public class WorkspaceSchemaFactory { } } - final FileSelection newSelection = hasDirectories ? fileSelection.minusDirectories(getFS()) : fileSelection; + final FileSelection newSelection = hasDirectories ? fileSelection.minusDirectories(fs) : fileSelection; if (newSelection == null) { return null; } for (final FormatMatcher matcher : fileMatchers) { - DrillTable table = matcher.isReadable(getFS(), newSelection, plugin, storageEngineName, schemaConfig.getUserName()); + DrillTable table = matcher.isReadable(fs, newSelection, plugin, storageEngineName, schemaConfig.getUserName()); if (table != null) { return table; } @@ -662,7 +632,7 @@ public class WorkspaceSchemaFactory { FormatMatcher matcher = null; try { for (FormatMatcher m : dropFileMatchers) { - if (m.isFileReadable(getFS(), file)) { + if (m.isFileReadable(fs, file)) { return m; } } @@ -685,8 +655,7 @@ public class WorkspaceSchemaFactory { * @throws IOException is case of problems accessing table files */ private boolean isHomogeneous(String tableName) throws IOException { - FileSelection fileSelection = - FileSelection.create(getFS(), config.getLocation(), tableName, config.allowAccessOutsideWorkspace()); + FileSelection fileSelection = FileSelection.create(fs, config.getLocation(), tableName, config.allowAccessOutsideWorkspace()); if (fileSelection == null) { throw UserException @@ -697,15 +666,15 @@ public class WorkspaceSchemaFactory { FormatMatcher matcher = null; Queue<FileStatus> listOfFiles = new LinkedList<>(); - listOfFiles.addAll(fileSelection.getStatuses(getFS())); + listOfFiles.addAll(fileSelection.getStatuses(fs)); while (!listOfFiles.isEmpty()) { FileStatus currentFile = listOfFiles.poll(); if (currentFile.isDirectory()) { - listOfFiles.addAll(DrillFileSystemUtil.listFiles(getFS(), currentFile.getPath(), true)); + listOfFiles.addAll(DrillFileSystemUtil.listFiles(fs, currentFile.getPath(), true)); } else { if (matcher != null) { - if (!matcher.isFileReadable(getFS(), currentFile)) { + if (!matcher.isFileReadable(fs, currentFile)) { return false; } } else { @@ -794,7 +763,7 @@ public class WorkspaceSchemaFactory { // Then look for files that start with this name and end in .drill. List<DotDrillFile> files = Collections.emptyList(); try { - files = DotDrillUtil.getDotDrills(getFS(), new Path(config.getLocation()), DotDrillType.VIEW); + files = DotDrillUtil.getDotDrills(fs, new Path(config.getLocation()), DotDrillType.VIEW); } catch (AccessControlException e) { if (!schemaConfig.getIgnoreAuthErrors()) { logger.debug(e.getMessage()); http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java index ce05543..60581a7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ischema/InfoSchemaBatchCreator.java @@ -33,7 +33,7 @@ public class InfoSchemaBatchCreator implements BatchCreator<InfoSchemaSubScan>{ @Override public ScanBatch getBatch(FragmentContext context, InfoSchemaSubScan config, List<RecordBatch> children) throws ExecutionSetupException { - RecordReader rr = config.getTable().getRecordReader(context.getFullRootSchema(), config.getFilter(), context.getOptions()); + RecordReader rr = config.getTable().getRecordReader(context.getRootSchema(), config.getFilter(), context.getOptions()); return new ScanBatch(config, context, Collections.singletonList(rr)); } } http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockBreakageStorage.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockBreakageStorage.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockBreakageStorage.java deleted file mode 100644 index f2c2d9f..0000000 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockBreakageStorage.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.store.mock; - -import org.apache.calcite.schema.SchemaPlus; -import org.apache.drill.exec.server.DrillbitContext; -import org.apache.drill.exec.store.SchemaConfig; - -import java.io.IOException; - -public class MockBreakageStorage extends MockStorageEngine { - - private boolean breakRegister; - - public MockBreakageStorage(MockStorageEngineConfig configuration, DrillbitContext context, String name) { - super(configuration, context, name); - breakRegister = false; - } - - public void setBreakRegister(boolean breakRegister) { - this.breakRegister = breakRegister; - } - - @Override - public void registerSchemas(SchemaConfig schemaConfig, SchemaPlus parent) throws IOException { - if (breakRegister) { - throw new IOException("mock breakRegister!"); - } - super.registerSchemas(schemaConfig, parent); - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java index 76363f3..90644b5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/mock/MockStorageEngine.java @@ -55,7 +55,7 @@ public class MockStorageEngine extends AbstractStoragePlugin { public MockStorageEngine(MockStorageEngineConfig configuration, DrillbitContext context, String name) { this.configuration = configuration; - this.schema = new MockSchema(this, name); + this.schema = new MockSchema(this); } @Override @@ -120,11 +120,6 @@ public class MockStorageEngine extends AbstractStoragePlugin { this.engine = engine; } - public MockSchema(MockStorageEngine engine, String name) { - super(ImmutableList.<String>of(), name); - this.engine = engine; - } - @Override public Table getTable(String name) { if (name.toLowerCase().endsWith(".json")) { http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java index f26848d..cf64b20 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/metadata/MetadataProvider.java @@ -543,7 +543,7 @@ public class MetadataProvider { private static <S> PojoRecordReader<S> getPojoRecordReader(final InfoSchemaTableType tableType, final InfoSchemaFilter filter, final DrillConfig config, final SchemaTreeProvider provider, final UserSession userSession) { final SchemaPlus rootSchema = - provider.createFullRootSchema(userSession.getCredentials().getUserName(), newSchemaConfigInfoProvider(config, userSession, provider)); + provider.createRootSchema(userSession.getCredentials().getUserName(), newSchemaConfigInfoProvider(config, userSession, provider)); return tableType.getRecordReader(rootSchema, filter, userSession.getOptions()); } http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSchema.java deleted file mode 100644 index 9282eed..0000000 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSchema.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.exec.physical.impl; - -import org.apache.drill.test.BaseDirTestWatcher; -import org.apache.drill.test.ClientFixture; -import org.apache.drill.test.ClusterFixture; -import org.apache.drill.test.ClusterMockStorageFixture; -import org.apache.drill.test.DrillTest; - -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; - -import static org.junit.Assert.assertTrue; - -public class TestSchema extends DrillTest { - - @ClassRule - public static final BaseDirTestWatcher dirTestWatcher = new BaseDirTestWatcher(); - - private static ClusterMockStorageFixture cluster; - private static ClientFixture client; - - @BeforeClass - public static void setup() throws Exception { - cluster = ClusterFixture.builder(dirTestWatcher).buildCustomMockStorage(); - boolean breakRegisterSchema = true; - // With a broken storage which will throw exception in regiterSchema, every query (even on other storage) - // shall fail if Drill is still loading all schemas (include the broken schema) before a query. - cluster.insertMockStorage("mock_broken", breakRegisterSchema); - cluster.insertMockStorage("mock_good", !breakRegisterSchema); - client = cluster.clientFixture(); - } - - @Test (expected = Exception.class) - public void testQueryBrokenStorage() throws Exception { - String sql = "SELECT id_i, name_s10 FROM `mock_broken`.`employees_5`"; - try { - client.queryBuilder().sql(sql).run(); - } catch (Exception ex) { - assertTrue(ex.getMessage().contains("VALIDATION ERROR: Schema")); - throw ex; - } - } - - @Test - public void testQueryGoodStorage() throws Exception { - String sql = "SELECT id_i, name_s10 FROM `mock_good`.`employees_5`"; - client.queryBuilder().sql(sql).run(); - } - - @Test - public void testQueryGoodStorageWithDefaultSchema() throws Exception { - String use_dfs = "use dfs.tmp"; - client.queryBuilder().sql(use_dfs).run(); - String sql = "SELECT id_i, name_s10 FROM `mock_good`.`employees_5`"; - client.queryBuilder().sql(sql).run(); - } - - @Test (expected = Exception.class) - public void testUseBrokenStorage() throws Exception { - try { - String use_dfs = "use mock_broken"; - client.queryBuilder().sql(use_dfs).run(); - } catch(Exception ex) { - assertTrue(ex.getMessage().contains("VALIDATION ERROR: Schema")); - throw ex; - } - } - -} http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java index dfd63de..82bcf75 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterFixtureBuilder.java @@ -282,8 +282,4 @@ public class ClusterFixtureBuilder { public ClusterFixture build() { return new ClusterFixture(this); } - - public ClusterMockStorageFixture buildCustomMockStorage() { - return new ClusterMockStorageFixture(this); - } } http://git-wip-us.apache.org/repos/asf/drill/blob/450e6709/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java b/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java deleted file mode 100644 index 54d7bf0..0000000 --- a/exec/java-exec/src/test/java/org/apache/drill/test/ClusterMockStorageFixture.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.drill.test; - -import org.apache.drill.exec.server.Drillbit; -import org.apache.drill.exec.store.StoragePluginRegistry; -import org.apache.drill.exec.store.StoragePluginRegistryImpl; -import org.apache.drill.exec.store.mock.MockBreakageStorage; -import org.apache.drill.exec.store.mock.MockStorageEngineConfig; - -public class ClusterMockStorageFixture extends ClusterFixture { - ClusterMockStorageFixture(ClusterFixtureBuilder builder) { - super(builder); - - } - - /** - * This should be called after bits are started - * @param name nthe mock storage name we are going to create - */ - public void insertMockStorage(String name, boolean breakRegisterSchema) { - for (Drillbit bit : drillbits()) { - - // Bit name and registration. - final StoragePluginRegistry pluginRegistry = bit.getContext().getStorage(); - MockStorageEngineConfig config = MockStorageEngineConfig.INSTANCE; - @SuppressWarnings("resource") - MockBreakageStorage plugin = new MockBreakageStorage( - MockStorageEngineConfig.INSTANCE, bit.getContext(), name); - ((StoragePluginRegistryImpl) pluginRegistry).definePlugin(name, config, plugin); - - plugin.setBreakRegister(breakRegisterSchema); - } - } - -}
