LENS-1504 : Authorization usinf Apache ranger in Lens.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/e39dec5f Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/e39dec5f Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/e39dec5f Branch: refs/heads/master Commit: e39dec5f86204a1bb4e5b60be136d59793db859f Parents: 11885b9 Author: Rajitha R <[email protected]> Authored: Mon Aug 13 14:53:14 2018 +0530 Committer: Rajitha.R <[email protected]> Committed: Mon Aug 13 14:53:14 2018 +0530 ---------------------------------------------------------------------- lens-api/src/main/resources/lens-errors.conf | 2 +- .../apache/lens/client/LensMetadataClient.java | 1 - lens-cube/pom.xml | 37 +++- .../cube/authorization/AuthorizationUtil.java | 66 +++++++ .../authorization/RangerLensAuthorizer.java | 100 ++++++++++ .../cube/authorization/RangerLensResource.java | 48 +++++ .../lens/cube/metadata/AbstractBaseTable.java | 16 ++ .../lens/cube/metadata/CubeFactTable.java | 1 - .../lens/cube/metadata/CubeMetastoreClient.java | 185 ++++++++++++++----- .../lens/cube/metadata/MetastoreConstants.java | 2 + .../lens/cube/metadata/MetastoreUtil.java | 4 + .../lens/cube/parse/CubeQueryRewriter.java | 3 + .../cube/parse/QueryAuthorizationResolver.java | 87 +++++++++ lens-cube/src/main/resources/ranger-sample.json | 100 ++++++++++ .../src/main/resources/hivedriver-default.xml | 12 ++ .../src/main/resources/jdbcdriver-default.xml | 12 ++ lens-examples/src/main/resources/dim_table.xml | 1 - .../apache/lens/lib/query/FileSystemUtil.java | 55 ++++++ .../apache/lens/lib/query/ZipFileFormatter.java | 10 +- .../lens/server/api/LensConfConstants.java | 91 ++++++++- .../apache/lens/server/api/LensErrorInfo.java | 28 +++ .../lens/server/api/SessionValidator.java | 2 + .../server/api/authorization/ActionType.java | 36 ++++ .../server/api/authorization/Authorizer.java | 33 ++++ .../api/authorization/DefaultAuthorizer.java | 30 +++ .../api/authorization/LensPrivilegeObject.java | 46 +++++ .../api/metastore/CubeMetastoreService.java | 1 - .../server/api/query/AbstractQueryContext.java | 10 +- .../server/api/query/QueryExecutionService.java | 15 ++ .../save/exception/PrivilegeException.java | 4 + .../server/api/user/UserGroupConfigLoader.java | 33 ++++ .../api/user/UserGroupLoaderException.java | 57 ++++++ .../org/apache/lens/server/BaseLensService.java | 32 +++- .../org/apache/lens/server/LensServices.java | 2 + .../lens/server/error/LensServerErrorCode.java | 3 +- .../lens/server/query/LensPersistentResult.java | 3 +- .../server/query/QueryExecutionServiceImpl.java | 35 +++- .../lens/server/query/QueryServiceResource.java | 28 +++ .../usergroup/FixedUserGroupConfigLoader.java | 51 +++++ .../user/usergroup/UserGroupLoaderFactory.java | 136 ++++++++++++++ .../src/main/resources/lensserver-default.xml | 16 ++ .../src/main/resources/ranger-lens-audit.xml | 20 ++ .../src/main/resources/ranger-lens-security.xml | 85 +++++++++ .../src/main/resources/ranger-policymgr-ssl.xml | 20 ++ pom.xml | 105 +++++++++++ src/site/apt/admin/config.apt | 44 +++-- src/site/apt/admin/hivedriver-config.apt | 30 +-- src/site/apt/admin/jdbcdriver-config.apt | 74 ++++---- 48 files changed, 1674 insertions(+), 138 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-api/src/main/resources/lens-errors.conf ---------------------------------------------------------------------- diff --git a/lens-api/src/main/resources/lens-errors.conf b/lens-api/src/main/resources/lens-errors.conf index fafd655..9001a45 100644 --- a/lens-api/src/main/resources/lens-errors.conf +++ b/lens-api/src/main/resources/lens-errors.conf @@ -25,7 +25,7 @@ BAD_REQUEST = 400 NOT_FOUND = 404 -UNAUTHORIZED = 401 +UNAUTHORIZED = 403 GONE = 410 TOO_MANY_REQUESTS = 429 INTERNAL_SERVER_ERROR = 500 http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java ---------------------------------------------------------------------- diff --git a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java index 7b37769..48e1208 100644 --- a/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java +++ b/lens-client/src/main/java/org/apache/lens/client/LensMetadataClient.java @@ -297,7 +297,6 @@ public class LensMetadataClient { return storages.getElements(); } - public APIResult createNewStorage(XStorage storage) { WebTarget target = getMetastoreWebTarget(); return translate(target.path("storages") http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/pom.xml ---------------------------------------------------------------------- diff --git a/lens-cube/pom.xml b/lens-cube/pom.xml index deaeb36..fca0106 100644 --- a/lens-cube/pom.xml +++ b/lens-cube/pom.xml @@ -34,12 +34,47 @@ <description>OLAP cube queries</description> <dependencies> + + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-common</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-client</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.core</groupId> + <artifactId>jersey-server</artifactId> + </dependency> + <dependency> + <groupId>org.glassfish.jersey.containers</groupId> + <artifactId>jersey-container-grizzly2-servlet</artifactId> + </dependency> + + <dependency> + <groupId>org.glassfish.jersey.containers</groupId> + <artifactId>jersey-container-servlet-core</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-cred</artifactId> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-audit</artifactId> + </dependency> <dependency> <groupId>org.apache.lens</groupId> <artifactId>lens-server-api</artifactId> <version>${project.version}</version> </dependency> - <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-exec</artifactId> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/authorization/AuthorizationUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/authorization/AuthorizationUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/authorization/AuthorizationUtil.java new file mode 100644 index 0000000..ccd46a3 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/authorization/AuthorizationUtil.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.authorization; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.authorization.ActionType; +import org.apache.lens.server.api.authorization.Authorizer; +import org.apache.lens.server.api.authorization.LensPrivilegeObject; +import org.apache.lens.server.api.error.LensException; +import org.apache.lens.server.api.query.save.exception.PrivilegeException; + +import org.apache.hadoop.conf.Configuration; + +/* +Helper class for all Authorization needs +*/ +public class AuthorizationUtil { + + private AuthorizationUtil(){} + + public static boolean isAuthorized(Authorizer authorizer, String tableName, + LensPrivilegeObject.LensPrivilegeObjectType privilegeObjectType, ActionType actionType, Configuration configuration) + throws LensException { + return isAuthorized(authorizer, tableName, null, privilegeObjectType, actionType, configuration); + } + + public static boolean isAuthorized(Authorizer authorizer, String tableName, String colName, + LensPrivilegeObject.LensPrivilegeObjectType privilegeObjectType, ActionType actionType, Configuration configuration) + throws LensException { + String user = null; + Set<String> userGroups = new HashSet<>(); + if (configuration.getBoolean(LensConfConstants.USER_NAME_BASED_AUTHORIZATION, + LensConfConstants.DEFAULT_USER_NAME_AUTHORIZATION)){ + user = configuration.get(LensConfConstants.SESSION_LOGGEDIN_USER); + } + if (configuration.getBoolean(LensConfConstants.USER_GROUPS_BASED_AUTHORIZATION, + LensConfConstants.DEFAULT_USER_GROUPS_AUTHORIZATION)) { + userGroups = (Set<String>) + configuration.getTrimmedStringCollection(LensConfConstants.SESSION_USER_GROUPS); + } + LensPrivilegeObject lp = new LensPrivilegeObject(privilegeObjectType, tableName, colName); + if (!authorizer.authorize(lp, actionType, user, userGroups)) { + throw new PrivilegeException(privilegeObjectType.toString(), tableName, actionType.toString()); + } + return true; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensAuthorizer.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensAuthorizer.java b/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensAuthorizer.java new file mode 100644 index 0000000..2fe08ef --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensAuthorizer.java @@ -0,0 +1,100 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.authorization; + +import java.util.Set; + +import org.apache.lens.server.api.authorization.ActionType; +import org.apache.lens.server.api.authorization.Authorizer; +import org.apache.lens.server.api.authorization.LensPrivilegeObject; + +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.policyengine.RangerAccessRequest; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.service.RangerBasePlugin; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +// Apache Ranger implementation for Authorization in Lens + +@Slf4j +public class RangerLensAuthorizer implements Authorizer { + + @Getter + private RangerBasePlugin rangerBasePlugin; + + RangerLensAuthorizer() { + this.init(); + } + + public void init() { + rangerBasePlugin = new RangerBasePlugin("lens", "lens"); + rangerBasePlugin.setResultProcessor(new RangerDefaultAuditHandler()); + rangerBasePlugin.init(); + } + + @Override + public boolean authorize(LensPrivilegeObject lensPrivilegeObject, ActionType accessType, String user, + Set<String> userGroups) { + + log.info("==> Lens Ranger Authorize User : "+ user + "User groups : " + userGroups + " Accesstype : " + + accessType + "Object : "+ lensPrivilegeObject.getTable()); + + RangerLensResource rangerLensResource = getLensResource(lensPrivilegeObject); + + boolean res = false; + + if (rangerLensResource != null) { + RangerAccessRequest rangerAccessRequest = new RangerAccessRequestImpl(rangerLensResource, + accessType.toString().toLowerCase(), user, userGroups); + RangerAccessResult rangerAccessResult = getRangerBasePlugin().isAccessAllowed(rangerAccessRequest); + res = rangerAccessResult != null && rangerAccessResult.getIsAllowed(); + } + + log.info("<== Lens Ranger Authorize User : "+ user + " User groups : " + userGroups + " Accesstype : "+ accessType + + " Object : "+ lensPrivilegeObject.getTable() + " Access : "+ res); + + return res; + } + + private RangerLensResource getLensResource(LensPrivilegeObject lensPrivilegeObject) { + + RangerLensResource lensResource = null; + switch (lensPrivilegeObject.getObjectType()) { + case COLUMN: + lensResource = new RangerLensResource(LensObjectType.COLUMN, lensPrivilegeObject.getTable(), + lensPrivilegeObject.getColumn()); + break; + case DATABASE: + lensResource = new RangerLensResource(LensObjectType.TABLE, lensPrivilegeObject.getTable(), null); + break; + + case NONE: + default: + break; + } + return lensResource; + } + + enum LensObjectType {NONE, TABLE, COLUMN} + + ; +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensResource.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensResource.java b/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensResource.java new file mode 100644 index 0000000..c449424 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/authorization/RangerLensResource.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.authorization; + +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; + +// Ranger equivalent for a lens resource + +class RangerLensResource extends RangerAccessResourceImpl { + + private static final String KEY_TABLE = "table"; + private static final String KEY_COLUMN = "column"; + + RangerLensResource(RangerLensAuthorizer.LensObjectType objectType, String table, String column) { + + switch(objectType) { + + case COLUMN: + setValue(KEY_TABLE, table); + setValue(KEY_COLUMN, column); + break; + + case TABLE: + setValue(KEY_TABLE, table); + break; + + case NONE: + default: + break; + } + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java index 5543308..e7c109f 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/AbstractBaseTable.java @@ -158,6 +158,22 @@ public abstract class AbstractBaseTable extends AbstractCubeTable { return getExpressionByName(column); } + + + /** + * Return restricted columns of the table, which can be specified by property + * MetastoreUtil.getRestrictedColumnsKey(getName()) + * + * @return set of restricted columns + */ + public Set<String> getRestrictedColumns() { + String restrictedColsStr = + MetastoreUtil.getNamedStringValue(getProperties(), MetastoreUtil.getRestrictedColumnsKey(getName())); + log.info("Restricted cols : "+ restrictedColsStr + " for table : "+ this.getName()); + return restrictedColsStr == null ? new HashSet<>() : new HashSet<>(Arrays.asList(StringUtils.split(restrictedColsStr + .toLowerCase(), ','))); + } + /** * Alters the expression if already existing or just adds if it is new expression. * http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java index 43e827a..8f3f4aa 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java @@ -48,7 +48,6 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable { this.storagePrefixUpdatePeriodMap = getUpdatePeriodMap(getName(), getProperties()); } - public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns, Map<String, Set<UpdatePeriod>> storageUpdatePeriods) { this(cubeName, factName, columns, storageUpdatePeriods, 0L, new HashMap<String, String>()); http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java index 3952696..e6afcff 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java @@ -29,12 +29,19 @@ import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.lens.api.metastore.*; + +import org.apache.lens.cube.authorization.AuthorizationUtil; import org.apache.lens.cube.error.LensCubeErrorCode; import org.apache.lens.cube.metadata.Storage.LatestInfo; import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo; import org.apache.lens.cube.metadata.timeline.PartitionTimeline; import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory; + import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.authorization.ActionType; +import org.apache.lens.server.api.authorization.Authorizer; +import org.apache.lens.server.api.authorization.LensPrivilegeObject; + import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.DataCompletenessChecker; import org.apache.lens.server.api.util.LensUtil; @@ -110,22 +117,50 @@ public class CubeMetastoreClient { private Boolean isDataCompletenessCheckEnabled; + private Boolean isAuthorizationCheckEnabled; + + private Authorizer authorizer; + public DataCompletenessChecker getCompletenessChecker() { if (completenessChecker == null) { completenessChecker = ReflectionUtils.newInstance(config.getClass(LensConfConstants.COMPLETENESS_CHECKER_CLASS, - LensConfConstants.DEFAULT_COMPLETENESS_CHECKER, DataCompletenessChecker.class), this.config); + LensConfConstants.DEFAULT_COMPLETENESS_CHECKER, DataCompletenessChecker.class), this.config); } return completenessChecker; } + public Authorizer getAuthorizer() { + if (authorizer == null) { + authorizer = ReflectionUtils.newInstance(config.getClass(MetastoreConstants.AUTHORIZER_CLASS, + LensConfConstants.DEFAULT_AUTHORIZER, Authorizer.class), this.config); + } + return authorizer; + } + public boolean isDataCompletenessCheckEnabled() { if (isDataCompletenessCheckEnabled == null) { isDataCompletenessCheckEnabled = config.getBoolean(LensConfConstants.ENABLE_DATACOMPLETENESS_CHECK, - LensConfConstants.DEFAULT_ENABLE_DATACOMPLETENESS_CHECK); + LensConfConstants.DEFAULT_ENABLE_DATACOMPLETENESS_CHECK); } return isDataCompletenessCheckEnabled; } + private boolean isAuthorizationEnabled() { + if (isAuthorizationCheckEnabled == null) { + isAuthorizationCheckEnabled = config.getBoolean(LensConfConstants.ENABLE_METASTORE_SCHEMA_AUTHORIZATION_CHECK, + LensConfConstants.DEFAULT_ENABLE_METASTORE_SCHEMA_AUTHORIZATION_CHECK); + } + return isAuthorizationCheckEnabled; + } + + private void checkIfAuthorized() throws LensException { + if (isAuthorizationEnabled()) { + String currentdb = SessionState.get().getCurrentDatabase(); + AuthorizationUtil.isAuthorized(getAuthorizer(), currentdb, + LensPrivilegeObject.LensPrivilegeObjectType.DATABASE, ActionType.UPDATE, getConf()); + } + } + /** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); @@ -306,6 +341,7 @@ public class CubeMetastoreClient { Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties, Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap) throws LensException { + checkIfAuthorized(); CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight, properties, storageUpdatePeriodMap); createCubeTable(factTable, storageTableDescs); @@ -314,11 +350,12 @@ public class CubeMetastoreClient { } - public <T extends Equals & HashCode & ToString> void createEntity(T entity) throws LensException { + public <T extends Equals & HashCode & ToString> void createEntity(T entity) + throws LensException { if (entity instanceof XStorage) { createStorage((XStorage) entity); - } else if (entity instanceof XCube) { - createCube((XCube)entity); + } else if (entity instanceof XCube) { + createCube((XCube) entity); } else if (entity instanceof XDimension) { createDimension((XDimension) entity); } else if (entity instanceof XFact) { @@ -328,7 +365,7 @@ public class CubeMetastoreClient { } else if (entity instanceof XSegmentation) { createSegmentation((XSegmentation) entity); } else { - throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: "+ entity.getClass()); + throw new LensException("Unable to create entity " + entity + " as it's unrecognizable: " + entity.getClass()); } } @@ -336,7 +373,7 @@ public class CubeMetastoreClient { throws LensException, HiveException { if (entity instanceof XStorage) { alterStorage((XStorage) entity); - } else if (entity instanceof XCube) { + } else if (entity instanceof XCube) { alterCube((XCube)entity); } else if (entity instanceof XDimension) { alterDimension((XDimension) entity); @@ -380,7 +417,7 @@ public class CubeMetastoreClient { public void createVirtualFactTable(String cubeName, String virtualFactName, String sourceFactName, Double weight, Map<String, String> properties) throws LensException { FactTable sourceFact = getFactTable(sourceFactName); - + checkIfAuthorized(); Optional<Double> optionalWeight = Optional.fromNullable(weight); CubeVirtualFactTable factTable = new CubeVirtualFactTable(cubeName, virtualFactName, @@ -571,7 +608,7 @@ public class CubeMetastoreClient { .computeIfAbsent(timeLineKey, s -> new TreeMap<>()) .computeIfAbsent(updatePeriod, k -> new CaseInsensitiveStringHashMap<>()) .computeIfAbsent(partitionColumn, c -> PartitionTimelineFactory.get( - CubeMetastoreClient.this, storagTableName, updatePeriod, c)); + CubeMetastoreClient.this, storagTableName, updatePeriod, c)); } /** check partition existence in the appropriate timeline if it exists */ @@ -588,8 +625,8 @@ public class CubeMetastoreClient { public PartitionTimeline get(String fact, String storage, UpdatePeriod updatePeriod, String partCol) throws HiveException, LensException { return get(fact, storage) != null && get(fact, storage).get(updatePeriod) != null - && get(fact, storage).get(updatePeriod).get(partCol) != null ? get(fact, storage).get(updatePeriod) - .get(partCol) : null; + && get(fact, storage).get(updatePeriod).get(partCol) != null ? get(fact, storage).get(updatePeriod) + .get(partCol) : null; } /** @@ -645,7 +682,7 @@ public class CubeMetastoreClient { * Get the instance of {@link CubeMetastoreClient} corresponding to {@link HiveConf} * * @param conf conf - * @return CubeMetastoreClient instance + * @return CubeMetastoreClient instance * @throws HiveException */ public static CubeMetastoreClient getInstance(HiveConf conf) throws HiveException { @@ -713,6 +750,7 @@ public class CubeMetastoreClient { } public void createStorage(Storage storage) throws LensException { + checkIfAuthorized(); createCubeHiveTable(storage); // do a get to update cache getStorage(storage.getName()); @@ -731,6 +769,7 @@ public class CubeMetastoreClient { * @throws LensException */ public void createCube(CubeInterface cube) throws LensException { + checkIfAuthorized(); createCubeHiveTable((AbstractCubeTable) cube); // do a get to update cache getCube(cube.getName()); @@ -816,6 +855,7 @@ public class CubeMetastoreClient { public void createDimension(XDimension dim) throws LensException { createDimension(JAXBUtils.dimensionFromXDimension(dim)); } + /** * Create dimension in metastore defined by {@link Dimension} object * @@ -823,6 +863,7 @@ public class CubeMetastoreClient { * @throws LensException */ public void createDimension(Dimension dim) throws LensException { + checkIfAuthorized(); createCubeHiveTable(dim); // do a get to update cache getDimension(dim.getName()); @@ -862,6 +903,7 @@ public class CubeMetastoreClient { Map<String, StorageTableDesc> storageTableDescs) throws LensException { CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight, properties); + checkIfAuthorized(); createCubeTable(factTable, storageTableDescs); // do a get to update cache getFactTable(factName); @@ -877,9 +919,9 @@ public class CubeMetastoreClient { * @throws LensException */ public void createSegmentation(String baseCubeName, String segmentationName, Set<Segment> segments, - double weight, Map<String, String> properties) throws LensException { + double weight, Map<String, String> properties) throws LensException { Segmentation cubeSeg = - new Segmentation(baseCubeName, segmentationName, segments, weight, properties); + new Segmentation(baseCubeName, segmentationName, segments, weight, properties); createSegmentation(cubeSeg); // do a get to update cache getSegmentation(segmentationName); @@ -897,6 +939,7 @@ public class CubeMetastoreClient { createCubeDimensionTable(xDimTable.getDimensionName(), xDimTable.getTableName(), columns, xDimTable.getWeight(), updatePeriodMap, properties, storageDesc); } + /** * Create a cube dimension table * @@ -914,6 +957,7 @@ public class CubeMetastoreClient { throws LensException { CubeDimensionTable dimTable = new CubeDimensionTable(dimName, dimTblName, columns, weight, storageNames, properties); + checkIfAuthorized(); createCubeTable(dimTable, storageTableDescs); // do a get to update cache getDimensionTable(dimTblName); @@ -934,6 +978,7 @@ public class CubeMetastoreClient { public void createCubeDimensionTable(String dimName, String dimTblName, List<FieldSchema> columns, double weight, Map<String, UpdatePeriod> dumpPeriods, Map<String, String> properties, Map<String, StorageTableDesc> storageTableDescs) throws LensException { + checkIfAuthorized(); CubeDimensionTable dimTable = new CubeDimensionTable(dimName, dimTblName, columns, weight, dumpPeriods, properties); createCubeTable(dimTable, storageTableDescs); // do a get to update cache @@ -968,8 +1013,10 @@ public class CubeMetastoreClient { cubeSeg.getWeight(), JAXBUtils.mapFromXProperties(cubeSeg.getProperties())); } + public void createSegmentation(Segmentation cubeSeg) throws LensException { + checkIfAuthorized(); // create virtual cube table in metastore createCubeHiveTable(cubeSeg); } @@ -986,6 +1033,7 @@ public class CubeMetastoreClient { public void addStorage(CubeFactTable fact, String storage, Set<UpdatePeriod> updatePeriods, Map<String, StorageTableDesc> storageTableDescs, Map<UpdatePeriod, String> updatePeriodStoragePrefix) throws LensException { + checkIfAuthorized(); fact.addStorage(storage, updatePeriods, updatePeriodStoragePrefix); for (Map.Entry entry : storageTableDescs.entrySet()) { createOrAlterStorageHiveTable(getTableWithTypeFailFast(fact.getName(), CubeTableType.FACT), @@ -1006,6 +1054,7 @@ public class CubeMetastoreClient { */ public void addStorage(CubeDimensionTable dim, String storage, UpdatePeriod dumpPeriod, StorageTableDesc storageTableDesc) throws LensException { + checkIfAuthorized(); dim.alterSnapshotDumpPeriod(storage, dumpPeriod); createOrAlterStorageHiveTable(getTableWithTypeFailFast(dim.getName(), CubeTableType.DIM_TABLE), storage, storageTableDesc); @@ -1054,7 +1103,8 @@ public class CubeMetastoreClient { * @throws LensException */ private List<Partition> addPartitions(String factOrDimTable, String storageName, UpdatePeriod updatePeriod, - List<StoragePartitionDesc> storagePartitionDescs, CubeTableType type) throws HiveException, LensException { + List<StoragePartitionDesc> storagePartitionDescs, CubeTableType type) + throws HiveException, LensException { String storageTableName = getStorageTableName(factOrDimTable, storageName, updatePeriod); if (type == CubeTableType.DIM_TABLE) { // Adding partition in dimension table. @@ -1079,12 +1129,12 @@ public class CubeMetastoreClient { List<Partition> partsAdded = new ArrayList<>(); // first update in memory, then add to hive table's partitions. delete is reverse. partitionTimelineCache.updateForAddition(factOrDimTable, storageName, updatePeriod, - getTimePartSpecs(storagePartitionDescs, getStorageTableStartDate(storageTableName, - getFactTable(factOrDimTable)), getStorageTableEndDate(storageTableName, getFactTable(factOrDimTable)))); + getTimePartSpecs(storagePartitionDescs, getStorageTableStartDate(storageTableName, + getFactTable(factOrDimTable)), getStorageTableEndDate(storageTableName, getFactTable(factOrDimTable)))); // Adding partition in fact table. if (storagePartitionDescs.size() > 0) { partsAdded = getStorage(storageName).addPartitions(getClient(), factOrDimTable, updatePeriod, - storagePartitionDescs, null, storageTableName); + storagePartitionDescs, null, storageTableName); } // update hive table alterTablePartitionCache((Storage.getPrefix(storageName) + factOrDimTable).toLowerCase(), updatePeriod, @@ -1124,7 +1174,7 @@ public class CubeMetastoreClient { private Map<String, TreeSet<Date>> getTimePartSpecs(List<StoragePartitionDesc> storagePartitionDescs, - Date storageStartDate, Date storageEndDate) throws LensException { + Date storageStartDate, Date storageEndDate) throws LensException { Date now = new Date(); Map<String, HashSet<Date>> skippedParts = Maps.newHashMap(); Map<String, TreeSet<Date>> timeSpecs = Maps.newHashMap(); @@ -1138,7 +1188,7 @@ public class CubeMetastoreClient { // check whether partition falls between storage table start_time and // end_time or d+2, in such case partition is eligible for registration. if ((entry.getValue().compareTo(storageStartDate) >= 0 && entry.getValue().compareTo(storageEndDate) < 0) - && entry.getValue().compareTo(DateUtil.resolveRelativeDate("now +2 days", now)) < 0) { + && entry.getValue().compareTo(DateUtil.resolveRelativeDate("now +2 days", now)) < 0) { timeSpecs.get(entry.getKey()).add(entry.getValue()); } else { if (!skippedParts.containsKey(entry.getKey())) { @@ -1153,7 +1203,7 @@ public class CubeMetastoreClient { } if (!skippedParts.isEmpty()) { log.info("List of partitions skipped : {}, because they fall before fact start time " - + "or after end time or they are future partitions", skippedParts); + + "or after end time or they are future partitions", skippedParts); } return timeSpecs; } @@ -1432,8 +1482,8 @@ public class CubeMetastoreClient { } public boolean factPartitionExists(String factName, String storageName, UpdatePeriod updatePeriod, - Map<String, Date> partitionTimestamp, - Map<String, String> partSpec) throws HiveException, LensException { + Map<String, Date> partitionTimestamp, + Map<String, String> partSpec) throws HiveException, LensException { String storageTableName = getStorageTableName(factName, storageName, updatePeriod); return partitionExists(storageTableName, updatePeriod, partitionTimestamp, partSpec); } @@ -1506,7 +1556,7 @@ public class CubeMetastoreClient { throws HiveException, LensException { String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storageName); return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName), - partitionTimestamps); + partitionTimestamps); } boolean latestPartitionExists(String factOrDimTblName, String storageName, String latestPartCol) @@ -1561,9 +1611,11 @@ public class CubeMetastoreClient { } return ret == null ? new ArrayList<String>() : ret; } + public Table getTableWithTypeFailFast(String tableName, CubeTableType type) throws LensException { return getTableWithType(tableName, type, true); } + public Table getTableWithType(String tableName, CubeTableType type, boolean throwException) throws LensException { String typeName = type == null ? "nativetable" : type.name().toLowerCase(); Table table = getTable(tableName, false); @@ -1593,9 +1645,11 @@ public class CubeMetastoreClient { } return table; } + public Table getTable(String tableName) throws LensException { return getTable(tableName, true); } + public Table getTable(String tableName, boolean throwException) throws LensException { Table tbl; try { @@ -1745,7 +1799,7 @@ public class CubeMetastoreClient { * Is the hive table a cube table? * * @param tbl table - * @return whether it's a cube table or not + * @return whether it's a cube table or not */ boolean isCube(Table tbl) { String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); @@ -1756,9 +1810,9 @@ public class CubeMetastoreClient { * Is the hive table a dimension? * * @param tbl table - * @return whether the hive table is a dimension or not + * @return whether the hive table is a dimension or not */ - boolean isDimension(Table tbl) { + boolean isDimension(Table tbl) { String tableType = tbl.getParameters().get(MetastoreConstants.TABLE_TYPE_KEY); return CubeTableType.DIMENSION.name().equals(tableType); } @@ -1766,6 +1820,7 @@ public class CubeMetastoreClient { public XFact getXFactTable(String tableName) throws LensException { return getXFactTable(getFactTable(tableName)); } + public XFact getXFactTable(FactTable ft) throws LensException { XFact fact; @@ -1823,6 +1878,7 @@ public class CubeMetastoreClient { public XDimensionTable getXDimensionTable(String dimTable) throws LensException { return getXDimensionTable(getDimensionTable(dimTable)); } + public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { @@ -1839,6 +1895,7 @@ public class CubeMetastoreClient { } return dt; } + /** * Get {@link CubeDimensionTable} object corresponding to the name * @@ -1849,6 +1906,7 @@ public class CubeMetastoreClient { public CubeDimensionTable getDimensionTable(String tableName) throws LensException { return getDimensionTable(tableName, true); } + private CubeDimensionTable getDimensionTable(String tableName, boolean throwException) throws LensException { tableName = tableName.trim().toLowerCase(); @@ -1896,6 +1954,7 @@ public class CubeMetastoreClient { public Storage getStorage(String storageName) throws LensException { return getStorage(storageName, true); } + public Storage getStorage(String storageName, boolean throwException) throws LensException { storageName = storageName.trim().toLowerCase(); Storage storage = allStorages.get(storageName); @@ -1964,6 +2023,7 @@ public class CubeMetastoreClient { public Dimension getDimension(String tableName) throws LensException { return getDimension(tableName, true); } + private Dimension getDimension(String tableName, boolean throwException) throws LensException { if (tableName == null) { return null; @@ -1996,6 +2056,7 @@ public class CubeMetastoreClient { public FactTable getFactTable(String tableName) throws LensException { return getFactTable(tableName, true); } + private FactTable getFactTable(String tableName, boolean throwException) throws LensException { tableName = tableName.trim().toLowerCase(); FactTable fact = allFactTables.get(tableName); @@ -2003,14 +2064,14 @@ public class CubeMetastoreClient { synchronized (allFactTables) { if (!allFactTables.containsKey(tableName)) { Table tbl = getTableWithType(tableName, CubeTableType.FACT, throwException); - if (tbl != null){ + if (tbl != null) { String sourceFactName = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); if (sourceFactName != null) { fact = new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFactName)); if (factToVirtualFactMapping.get(sourceFactName) != null) { List<String> prevList = factToVirtualFactMapping.get(sourceFactName); prevList.add(tableName); - }else{ + } else { List<String> newList = new ArrayList<>(); newList.add(tableName); factToVirtualFactMapping.put(sourceFactName, newList); @@ -2042,6 +2103,7 @@ public class CubeMetastoreClient { public Segmentation getSegmentation(String segName) throws LensException { return getSegmentation(segName, true); } + public Segmentation getSegmentation(String segName, boolean throwException) throws LensException { segName = segName.trim().toLowerCase(); Segmentation seg = allSegmentations.get(segName); @@ -2164,7 +2226,7 @@ public class CubeMetastoreClient { List<Dimension> dims = new ArrayList<>(); try { for (String table : getAllHiveTableNames()) { - Dimension dim = getDimension(table, false); + Dimension dim = getDimension(table, false); if (dim != null) { dims.add(dim); } @@ -2454,6 +2516,7 @@ public class CubeMetastoreClient { ((XDerivedCube) cube).getParent()) : null; alterCube(cube.getName(), JAXBUtils.hiveCubeFromXCube(cube, parent)); } + /** * Alter cube specified by the name to new definition * @@ -2461,7 +2524,9 @@ public class CubeMetastoreClient { * @param cube The new cube definition {@link Cube} or {@link DerivedCube} * @throws HiveException */ - public void alterCube(String cubeName, CubeInterface cube) throws HiveException, LensException { + public void alterCube(String cubeName, CubeInterface cube) + throws HiveException, LensException { + checkIfAuthorized(); Table cubeTbl = getTableWithTypeFailFast(cubeName, CubeTableType.CUBE); alterCubeTable(cubeName, cubeTbl, (AbstractCubeTable) cube); if (enableCaching) { @@ -2479,7 +2544,9 @@ public class CubeMetastoreClient { alterDimension(newDim.getName(), JAXBUtils.dimensionFromXDimension(newDim)); } - public void alterDimension(String dimName, Dimension newDim) throws HiveException, LensException { + public void alterDimension(String dimName, Dimension newDim) + throws HiveException, LensException { + checkIfAuthorized(); Table tbl = getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION); alterCubeTable(dimName, tbl, newDim); if (enableCaching) { @@ -2496,7 +2563,10 @@ public class CubeMetastoreClient { public void alterStorage(XStorage storage) throws LensException, HiveException { alterStorage(storage.getName(), JAXBUtils.storageFromXStorage(storage)); } - public void alterStorage(String storageName, Storage storage) throws LensException, HiveException { + + public void alterStorage(String storageName, Storage storage) + throws LensException, HiveException { + checkIfAuthorized(); Table storageTbl = getTableWithTypeFailFast(storageName, CubeTableType.STORAGE); alterCubeTable(storageName, storageTbl, storage); if (enableCaching) { @@ -2523,7 +2593,8 @@ public class CubeMetastoreClient { * @throws LensException */ public void dropCube(String cubeName) throws LensException { - getTableWithTypeFailFast(cubeName, CubeTableType.CUBE); + checkIfAuthorized(); + getCube(getTableWithTypeFailFast(cubeName, CubeTableType.CUBE)); allCubes.remove(cubeName.trim().toLowerCase()); dropHiveTable(cubeName); } @@ -2536,6 +2607,7 @@ public class CubeMetastoreClient { */ public void dropDimension(String dimName) throws LensException { getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION); + checkIfAuthorized(); allDims.remove(dimName.trim().toLowerCase()); dropHiveTable(dimName); } @@ -2550,6 +2622,7 @@ public class CubeMetastoreClient { public void dropFact(String factName, boolean cascade) throws LensException { getTableWithTypeFailFast(factName, CubeTableType.FACT); FactTable fact = getFactTable(factName); + checkIfAuthorized(); if (cascade) { for (String storage : fact.getStorages()) { dropStorageFromFact(factName, storage, false); @@ -2602,6 +2675,7 @@ public class CubeMetastoreClient { public void dropSegmentation(String segName) throws LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); + checkIfAuthorized(); dropHiveTable(segName); allSegmentations.remove(segName.trim().toLowerCase()); } @@ -2615,13 +2689,14 @@ public class CubeMetastoreClient { */ public void dropStorageFromFact(String factName, String storage) throws LensException { CubeFactTable cft = getCubeFactTable(factName); + checkIfAuthorized(); dropHiveTablesForStorage(factName, storage); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); } - private void dropHiveTablesForStorage(String factName, String storage) throws LensException{ + private void dropHiveTablesForStorage(String factName, String storage) throws LensException { CubeFactTable cft = getCubeFactTable(factName); Set<String> droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { @@ -2633,6 +2708,7 @@ public class CubeMetastoreClient { droppedTables.add(storageTableName); } } + // updateFact will be false when fact is fully dropped private void dropStorageFromFact(String factName, String storage, boolean updateFact) throws LensException { @@ -2645,6 +2721,7 @@ public class CubeMetastoreClient { } } + /** * Drop a storage from dimension * @@ -2652,13 +2729,15 @@ public class CubeMetastoreClient { * @param storage storage * @throws HiveException */ - public void dropStorageFromDim(String dimTblName, String storage) throws HiveException, LensException { + public void dropStorageFromDim(String dimTblName, String storage) + throws HiveException, LensException { dropStorageFromDim(dimTblName, storage, true); } // updateDimTbl will be false when dropping dimTbl private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws LensException { + checkIfAuthorized(); CubeDimensionTable cdt = getDimensionTable(dimTblName); String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storage); dropHiveTable(storageTableName); @@ -2677,7 +2756,9 @@ public class CubeMetastoreClient { * @param cascade If true, will drop all the dimension storages * @throws HiveException */ - public void dropDimensionTable(String dimTblName, boolean cascade) throws LensException { + public void dropDimensionTable(String dimTblName, boolean cascade) + throws LensException { + checkIfAuthorized(); getTableWithTypeFailFast(dimTblName, CubeTableType.DIM_TABLE); CubeDimensionTable dim = getDimensionTable(dimTblName); if (cascade) { @@ -2712,10 +2793,11 @@ public class CubeMetastoreClient { * @throws HiveException */ public void alterCubeFactTable(String factTableName, FactTable cubeFactTable, - Map<String, StorageTableDesc> storageTableDescs, - Map<String, String> props) + Map<String, StorageTableDesc> storageTableDescs, + Map<String, String> props) throws HiveException, LensException { Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT); + checkIfAuthorized(); if (!props.isEmpty()) { cubeFactTable.getProperties().putAll(props); } @@ -2731,7 +2813,8 @@ public class CubeMetastoreClient { updateAllVirtualFacts(getFactTable(factTableName)); } - public void alterSegmentation(XSegmentation cubeSeg) throws LensException, HiveException { + public void alterSegmentation(XSegmentation cubeSeg) throws LensException, + HiveException { alterSegmentation(cubeSeg.getName(), segmentationFromXSegmentation(cubeSeg)); } @@ -2748,6 +2831,7 @@ public class CubeMetastoreClient { public void alterSegmentation(String segName, Segmentation seg) throws HiveException, LensException { getTableWithTypeFailFast(segName, CubeTableType.SEGMENTATION); + checkIfAuthorized(); if (!(getSegmentation(segName) == seg)) { dropSegmentation(segName); createSegmentation(seg); @@ -2765,11 +2849,11 @@ public class CubeMetastoreClient { if (enableCaching) { Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT); FactTable refreshedTable; - if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null){ + if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null) { String sourceFactName = factTbl.getParameters().get(getSourceFactNameKey(factTableName)); refreshedTable = new CubeVirtualFactTable(refreshTable(factTableName), getCubeFactTable(sourceFactName)); - }else { + } else { refreshedTable = new CubeFactTable(refreshTable(factTableName)); } allFactTables.put(factTableName.trim().toLowerCase(), refreshedTable); @@ -2817,11 +2901,14 @@ public class CubeMetastoreClient { allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName))); } } - public void alterCubeDimensionTable(XDimensionTable dimensionTable) throws LensException, HiveException { + + public void alterCubeDimensionTable(XDimensionTable dimensionTable) + throws LensException, HiveException { alterCubeDimensionTable(dimensionTable.getTableName(), JAXBUtils.cubeDimTableFromDimTable(dimensionTable), JAXBUtils.tableDescPrefixMapFromXStorageTables(dimensionTable.getStorageTables())); } + /** * Alter dimension table with new dimension definition and underlying storage tables as well * @@ -2830,7 +2917,9 @@ public class CubeMetastoreClient { * @throws HiveException */ public void alterCubeDimensionTable(String dimTableName, CubeDimensionTable cubeDimensionTable, - Map<String, StorageTableDesc> storageTableDescs) throws HiveException, LensException { + Map<String, StorageTableDesc> storageTableDescs) + throws HiveException, LensException { + checkIfAuthorized(); Table dimTbl = getTableWithTypeFailFast(dimTableName, CubeTableType.DIM_TABLE); alterCubeTable(dimTableName, dimTbl, cubeDimensionTable); if (storageTableDescs != null) { @@ -2862,7 +2951,7 @@ public class CubeMetastoreClient { public boolean isStorageTableCandidateForRange(String storageTableName, Date fromDate, Date toDate) throws LensException { List<Date> storageEndDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableEndTimesKey()); - for(Date endDate : storageEndDates) { + for (Date endDate : storageEndDates) { // endDate is exclusive if (endDate.before(fromDate) || endDate.equals(fromDate)) { log.debug("from date {} is after validity end time: {}, hence discarding {}", @@ -2872,7 +2961,7 @@ public class CubeMetastoreClient { } List<Date> storageStartDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableStartTimesKey()); - for(Date startDate : storageStartDates) { + for (Date startDate : storageStartDates) { // toDate is exclusive on the range if (startDate.after(toDate) || startDate.equals(toDate)) { log.debug("to date {} is before validity start time: {}, hence discarding {}", @@ -2887,7 +2976,7 @@ public class CubeMetastoreClient { public boolean isStorageTablePartitionACandidate(String storageTableName, Date partDate) throws LensException { List<Date> storageStartDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableStartTimesKey()); - for(Date startDate : storageStartDates) { + for (Date startDate : storageStartDates) { if (partDate.before(startDate)) { log.info("part date {} is before validity start time: {}, hence discarding {}", partDate, startDate, storageTableName); @@ -2896,7 +2985,7 @@ public class CubeMetastoreClient { } List<Date> storageEndDates = getStorageTimes(storageTableName, MetastoreUtil.getStoragetableEndTimesKey()); - for(Date endDate : storageEndDates) { + for (Date endDate : storageEndDates) { // end date should be exclusive if (partDate.after(endDate) || partDate.equals(endDate)) { log.info("part date {} is after validity end time: {}, hence discarding {}", http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java index 945fe26..160addb 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreConstants.java @@ -26,6 +26,7 @@ public final class MetastoreConstants { public static final String TABLE_TYPE_KEY = "cube.table.type"; public static final String CUBE_TABLE_PFX = "cube.table."; public static final String WEIGHT_KEY_SFX = ".weight"; + public static final String AUTHORIZER_CLASS = "authorizer.class"; public static final String BASE_KEY_PFX = "base."; public static final String EXPRESSIONS_LIST_SFX = ".expressions.list"; @@ -51,6 +52,7 @@ public final class MetastoreConstants { public static final String CUBE_NAME_SFX = ".cubename"; public static final String SOURCE_NAME_SFX = ".source"; public static final String VALID_COLUMNS_SFX = ".valid.columns"; + public static final String RESTRICTED_COLUMNS_SFX = ".restricted.columns"; public static final String FACT_AGGREGATED_PROPERTY = "cube.fact.is.aggregated"; public static final String FACT_ABSOLUTE_START_TIME = "cube.fact.absolute.start.time"; public static final String FACT_RELATIVE_START_TIME = "cube.fact.relative.start.time"; http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java index fbc37ac..44411f2 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java @@ -374,6 +374,10 @@ public class MetastoreUtil { return getFactKeyPrefix(name) + VALID_COLUMNS_SFX; } + public static String getRestrictedColumnsKey(String name) { + return getCubePrefix(name) + RESTRICTED_COLUMNS_SFX; + } + public static String getCubeTableWeightKey(String name) { return getCubeTableKeyPrefix(name) + WEIGHT_KEY_SFX; } http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java index 143b266..5ddc950 100644 --- a/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/CubeQueryRewriter.java @@ -146,6 +146,7 @@ public class CubeQueryRewriter { // Rewrite base trees (groupby, having, orderby, limit) using aliases rewriters.add(new AliasReplacer()); ExpressionResolver exprResolver = new ExpressionResolver(); + QueryAuthorizationResolver queryAuthorizationResolver = new QueryAuthorizationResolver(conf); DenormalizationResolver denormResolver = new DenormalizationResolver(); CandidateTableResolver candidateTblResolver = new CandidateTableResolver(conf); StorageTableResolver storageTableResolver = new StorageTableResolver(conf); @@ -155,6 +156,8 @@ public class CubeQueryRewriter { rewriters.add(exprResolver); // Phase 1 of denormResolver: De-normalized columns resolved rewriters.add(denormResolver); + // authorization check + rewriters.add(queryAuthorizationResolver); // Resolve time ranges rewriters.add(new TimerangeResolver()); // Phase 1 of candidateTblResolver: Resolve candidate storages and dimension tables for columns queried http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAuthorizationResolver.java ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAuthorizationResolver.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAuthorizationResolver.java new file mode 100644 index 0000000..78dd642 --- /dev/null +++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/QueryAuthorizationResolver.java @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.cube.parse; + +import java.util.*; + +import org.apache.lens.cube.authorization.AuthorizationUtil; +import org.apache.lens.cube.metadata.*; +import org.apache.lens.server.api.LensConfConstants; +import org.apache.lens.server.api.authorization.ActionType; +import org.apache.lens.server.api.authorization.Authorizer; +import org.apache.lens.server.api.authorization.LensPrivilegeObject; +import org.apache.lens.server.api.error.LensException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.util.ReflectionUtils; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class QueryAuthorizationResolver implements ContextRewriter { + + @Getter + private Authorizer authorizer; + @Getter + private Boolean isAuthorizationCheckEnabled; + + QueryAuthorizationResolver(Configuration conf) { + isAuthorizationCheckEnabled = conf.getBoolean(LensConfConstants.ENABLE_QUERY_AUTHORIZATION_CHECK, + LensConfConstants.DEFAULT_ENABLE_QUERY_AUTHORIZATION_CHECK); + authorizer = ReflectionUtils.newInstance( + conf.getClass(MetastoreConstants.AUTHORIZER_CLASS, LensConfConstants.DEFAULT_AUTHORIZER, Authorizer.class), + conf); + } + @Override + public void rewriteContext(CubeQueryContext cubeql) throws LensException { + + log.info("==> Query Authorization enabled : "+ isAuthorizationCheckEnabled); + if (isAuthorizationCheckEnabled) { + for (Map.Entry<String, Set<String>> entry : cubeql.getTblAliasToColumns().entrySet()) { + String alias = entry.getKey(); + // skip default alias + if (Objects.equals(alias, CubeQueryContext.DEFAULT_TABLE)) { + continue; + } + AbstractCubeTable tbl = cubeql.getCubeTableForAlias(alias); + Set<String> queriedColumns = entry.getValue(); + + Set<String> restrictedFieldsQueried = + getRestrictedColumnsFromQuery(((AbstractBaseTable) tbl).getRestrictedColumns(), queriedColumns); + log.info("Restricted queriedColumns queried : "+ restrictedFieldsQueried); + if (restrictedFieldsQueried != null && !restrictedFieldsQueried.isEmpty()) { + for (String col : restrictedFieldsQueried) { + AuthorizationUtil.isAuthorized(getAuthorizer(), tbl.getName(), col, + LensPrivilegeObject.LensPrivilegeObjectType.COLUMN, ActionType.SELECT, cubeql.getConf()); + } + } + } + } + log.info("<== Query Authorization enabled : "+ isAuthorizationCheckEnabled); + } + + /* + * Returns the intersection of queried and restricted columns of table + * */ + private static Set<String> getRestrictedColumnsFromQuery(Set<String> restrictedColumns, Set<String> queriedColumns){ + restrictedColumns.retainAll(queriedColumns); + return restrictedColumns; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-cube/src/main/resources/ranger-sample.json ---------------------------------------------------------------------- diff --git a/lens-cube/src/main/resources/ranger-sample.json b/lens-cube/src/main/resources/ranger-sample.json new file mode 100644 index 0000000..51e0e60 --- /dev/null +++ b/lens-cube/src/main/resources/ranger-sample.json @@ -0,0 +1,100 @@ +{ + "id":1, + "name": "lens", + "implClass": "org.apache.ranger.plugin.service.RangerDefaultService", + "label": "Sample Repository", + "description": "Sample Repository", + "guid": "test", + "resources": + [ + { + "itemId": 1, + "name": "table", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": true, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { "wildCard":true, "ignoreCase":false }, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Lens database/cube", + "description": "Lens database/cube" + }, + { + "itemId": 2, + "name": "column", + "type": "string", + "level": 30, + "parent": "table", + "mandatory": false, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard":true, "ignoreCase":true }, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Lens Column", + "description": "Lens Column" + } + ], + + "accessTypes": + [ + { + "itemId": 1, + "name": "create", + "label": "create" + }, + + { + "itemId": 2, + "name": "read", + "label": "read" + }, + { + "itemId": 3, + "name": "update", + "label": "update" + }, + + { + "itemId": 4, + "name": "delete", + "label": "delete" + }, + + { + "itemId": 5, + "name": "select", + "label": "select" + } + ], + + "configs": + [ + + ], + + "enums": + [ + + ], + + "contextEnrichers": + [ + + ], + + "policyConditions": + [ + + ] +} + http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-driver-hive/src/main/resources/hivedriver-default.xml ---------------------------------------------------------------------- diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml index 2776123..2a912ad 100644 --- a/lens-driver-hive/src/main/resources/hivedriver-default.xml +++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml @@ -197,4 +197,16 @@ <description>List of policy decider classes</description> </property> + <property> + <name>lens.cube.query.user.groups.authorization.enable</name> + <value>false</value> + <description>true if authorization is based on User Groups, false otherwise</description> + </property> + + <property> + <name>lens.cube.query.user.name.authorization.enable</name> + <value>false</value> + <description>true if authorization is based on User Name, false otherwise</description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml ---------------------------------------------------------------------- diff --git a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml index 2c69eef..c18d732 100644 --- a/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml +++ b/lens-driver-jdbc/src/main/resources/jdbcdriver-default.xml @@ -306,4 +306,16 @@ <description>List of classes to decide policies</description> </property> + <property> + <name>lens.cube.query.user.groups.authorization.enable</name> + <value>false</value> + <description>true if authorization is based on User Groups, false otherwise</description> + </property> + + <property> + <name>lens.cube.query.user.name.authorization.enable</name> + <value>false</value> + <description>true if authorization is based on User Name, false otherwise</description> + </property> + </configuration> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-examples/src/main/resources/dim_table.xml ---------------------------------------------------------------------- diff --git a/lens-examples/src/main/resources/dim_table.xml b/lens-examples/src/main/resources/dim_table.xml index 7659555..d25932d 100644 --- a/lens-examples/src/main/resources/dim_table.xml +++ b/lens-examples/src/main/resources/dim_table.xml @@ -43,6 +43,5 @@ <time_part_cols>dt</time_part_cols> </table_desc> </storage_table> - </storage_tables> </x_dimension_table> http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSystemUtil.java ---------------------------------------------------------------------- diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSystemUtil.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSystemUtil.java new file mode 100644 index 0000000..c4eeb8e --- /dev/null +++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/FileSystemUtil.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.lib.query; + +import java.io.IOException; +import java.security.PrivilegedExceptionAction; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.UserGroupInformation; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FileSystemUtil { + + private FileSystemUtil() {} + + public static FileSystem createFileSystem(String proxyUser, Path path) { + try { + UserGroupInformation ugi = UserGroupInformation.createProxyUser(proxyUser, UserGroupInformation.getLoginUser()); + return ugi.doAs((PrivilegedExceptionAction<FileSystem>) () -> open(path)); + } catch (IOException e) { + log.error("error initializing filesystem, giving up leadership", e); + throw new RuntimeException(e); + } catch (InterruptedException e) { + log.error("interrupted while initializing filesystem, giving up leadership", e); + throw new RuntimeException(e); + } + } + + public static FileSystem open(Path filePath) throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean("fs.automatic.close", false); + return filePath.getFileSystem(conf); + } + +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java ---------------------------------------------------------------------- diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java index 8e9859f..3ba4655 100644 --- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java +++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java @@ -23,6 +23,8 @@ import java.io.OutputStreamWriter; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import org.apache.lens.server.api.LensConfConstants; + import org.apache.commons.lang.StringUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -107,8 +109,12 @@ public class ZipFileFormatter extends AbstractFileFormatter { finalPath = new Path(pathStr, finalPathStr + ctx.getQueryHandle().toString() + ".zip"); tmpPath = new Path(pathStr, ctx.getQueryHandle().toString() + ".tmp.zip"); - fs = finalPath.getFileSystem(ctx.getConf()); - + if (ctx.getConf().getBoolean(LensConfConstants.READ_RESULT_FROM_HDFS, + LensConfConstants.DEFAULT_READ_RESULT_FROM_HDFS)) { + fs = FileSystemUtil.createFileSystem(ctx.getSubmittedUser(), new Path(pathStr)); + } else { + fs = finalPath.getFileSystem(ctx.getConf()); + } zipOut = new ZipOutputStream((fs.create(tmpPath))); ZipEntry zipEntry = new ZipEntry(getQueryResultFileName()); zipOut.putNextEntry(zipEntry); http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java index 58a235a..efaf5d2 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java @@ -21,6 +21,8 @@ package org.apache.lens.server.api; import javax.ws.rs.core.MediaType; import org.apache.lens.api.parse.Parser; +import org.apache.lens.server.api.authorization.Authorizer; +import org.apache.lens.server.api.authorization.DefaultAuthorizer; import org.apache.lens.server.api.error.LensException; import org.apache.lens.server.api.metastore.*; import org.apache.lens.server.api.query.DefaultDownloadResultUrlProvider; @@ -270,6 +272,11 @@ public final class LensConfConstants { public static final String SESSION_CLUSTER_USER = SESSION_PFX + "cluster.user"; /** + * The Constant SESSION_USER_GROUPS. + */ + public static final String SESSION_USER_GROUPS = SESSION_PFX + "user.groups"; + + /** * The Constant MAPRED_JOB_QUEUE_NAME. */ public static final String MAPRED_JOB_QUEUE_NAME = "mapred.job.queue.name"; @@ -290,12 +297,23 @@ public final class LensConfConstants { */ public static final String USER_RESOLVER_TYPE = SERVER_PFX + "user.resolver.type"; + // ldap user to user group for authorization checks + /** + * The Constant USER_GROUP_TYPE. + */ + public static final String USER_GROUP_TYPE = SERVER_PFX + "user.group.type"; + /** * The Constant USER_RESOLVER_FIXED_VALUE. */ public static final String USER_RESOLVER_FIXED_VALUE = SERVER_PFX + "user.resolver.fixed.value"; /** + * The Constant USER_GROUP_FIXED_VALUE. + */ + public static final String USER_GROUP_FIXED_VALUE = SERVER_PFX + "user.group.fixed.value"; + + /** * The Constant USER_RESOLVER_PROPERTYBASED_FILENAME. */ public static final String USER_RESOLVER_PROPERTYBASED_FILENAME = SERVER_PFX + "user.resolver.propertybased.filename"; @@ -316,6 +334,11 @@ public final class LensConfConstants { public static final String USER_RESOLVER_CUSTOM_CLASS = SERVER_PFX + "user.resolver.custom.class"; /** + * The Constant USER_GROUP_CUSTOM_CLASS. + */ + public static final String USER_GROUP_CUSTOM_CLASS = SERVER_PFX + "user.group.custom.class"; + + /** * The Constant USER_RESOLVER_CACHE_EXPIRY. */ public static final String USER_RESOLVER_CACHE_EXPIRY = SERVER_PFX + "user.resolver.cache.expiry"; @@ -374,6 +397,26 @@ public final class LensConfConstants { public static final String USER_RESOLVER_LDAP_SEARCH_FILTER = SERVER_PFX + "user.resolver.ldap.search.filter"; /** + * The Constant USER_AUTHORIZATION. + */ + public static final String USER_NAME_BASED_AUTHORIZATION = "lens.cube.query.user.name.authorization.enable"; + + /** + * The Constant USER_GROUPS_BASED_AUTHORIZATION. + */ + public static final String USER_GROUPS_BASED_AUTHORIZATION = "lens.cube.query.user.groups.authorization.enable"; + + /** + * The default USER_AUTHORIZATION. + */ + public static final Boolean DEFAULT_USER_NAME_AUTHORIZATION = false; + + /** + * The default USER_GROUPS_BASED_AUTHORIZATION. + */ + public static final Boolean DEFAULT_USER_GROUPS_AUTHORIZATION = false; + + /** * The Constant STORAGE_COST. */ public static final String STORAGE_COST = METASTORE_PFX + "table.storage.cost"; @@ -439,7 +482,7 @@ public final class LensConfConstants { return SERVER_PFX + featureName + WS_FEATURE_IMPL_SFX; } - /** + /** * Gets the WS listener impl conf key. * * @param listenerName the listener name @@ -715,6 +758,16 @@ public final class LensConfConstants { public static final String RESULT_FS_READ_URL = QUERY_PFX + "result.fs.read.url"; /** + * The Constant READ_RESULT_FROM_HDFS. + */ + public static final String READ_RESULT_FROM_HDFS = QUERY_PFX + "read.result.hdfs"; + + /** + * The Constant DEFAULT_READ_RESULT_FROM_HDFS. + */ + public static final Boolean DEFAULT_READ_RESULT_FROM_HDFS = false; + + /** * The Constant AUX_JARS. */ public static final String AUX_JARS = SESSION_PFX + "aux.jars"; @@ -1284,6 +1337,10 @@ public final class LensConfConstants { public static final Class<? extends DataCompletenessChecker> DEFAULT_COMPLETENESS_CHECKER = DefaultChecker.class.asSubclass(DataCompletenessChecker.class); + + public static final Class<? extends Authorizer> DEFAULT_AUTHORIZER = + DefaultAuthorizer.class.asSubclass(Authorizer.class); + /** * This property is to enable Data Completeness Checks while resolving partitions. */ @@ -1295,6 +1352,38 @@ public final class LensConfConstants { public static final boolean DEFAULT_ENABLE_DATACOMPLETENESS_CHECK = false; /** + * This property is to enable authorization checks while query planning. + */ + public static final String ENABLE_QUERY_AUTHORIZATION_CHECK = "lens.cube.metastore.enable.query.authorization.check"; + + /** + * Default Value of the config "lens.cube.metastore.enable.query.authorization.check" + */ + public static final boolean DEFAULT_ENABLE_QUERY_AUTHORIZATION_CHECK = false; + + /** + * This property is to enable authorization checks while downloading result. + */ + public static final String ENABLE_RESULT_DOWNLOAD_AUTHORIZATION_CHECK = + "lens.enable.result.download.authorization.check"; + + /** + * Default Value of the config "lens.enable.result.download.authorization.check" + */ + public static final boolean DEFAULT_ENABLE_RESULT_DOWNLOAD_AUTHORIZATION_CHECK = false; + + /** + * This property is to enable authorization checks for schema changes. + */ + public static final String ENABLE_METASTORE_SCHEMA_AUTHORIZATION_CHECK = + "lens.cube.metastore.enable.metastore.authorization.check"; + + /** + * Default Value of the config "lens.cube.metastore.enable.schema.authorization.check" + */ + public static final boolean DEFAULT_ENABLE_METASTORE_SCHEMA_AUTHORIZATION_CHECK = false; + + /** * This property is for setting static cost to driver */ public static final String DRIVER_QUERY_COST = DRIVER_PFX + "query.cost"; http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/LensErrorInfo.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensErrorInfo.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensErrorInfo.java index 449120d..21c3027 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensErrorInfo.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensErrorInfo.java @@ -31,4 +31,32 @@ public class LensErrorInfo { @Getter private String errorName; + @Override + public boolean equals(final Object o) { + + if (this == o) { + return true; + } + + if (!(o instanceof LensErrorInfo)) { + return false; + } + + LensErrorInfo e = (LensErrorInfo) o; + return errorCode == e.errorCode && errorWeight == e.errorWeight && errorName.equals(e.errorName); + } + + + @Override + public int hashCode() { + + final int PRIME = 59; + int result = 1; + + result = result * PRIME + errorCode; + result = result * PRIME + errorWeight; + result = result * PRIME + errorName.hashCode(); + return result; + } + } http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java index 1185274..6540319 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/SessionValidator.java @@ -23,4 +23,6 @@ import org.apache.lens.server.api.error.LensException; public interface SessionValidator { void validateSession(LensSessionHandle handle) throws LensException; + + void validateAndAuthorizeSession(LensSessionHandle handle, String userPrincipalName) throws LensException; } http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/ActionType.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/ActionType.java b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/ActionType.java new file mode 100644 index 0000000..c8c4a66 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/ActionType.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.authorization; + +import lombok.Getter; + +public enum ActionType { + CREATE, + READ, + UPDATE, + DELETE, + SELECT; + + @Getter + private String actionName; + + ActionType(){ + this.actionName = name().toLowerCase(); + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/Authorizer.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/Authorizer.java b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/Authorizer.java new file mode 100644 index 0000000..502a8a7 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/Authorizer.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.lens.server.api.authorization; + +import java.util.Set; + +public interface Authorizer { + /** + * @param lensPrivilegeObject the privilege object + * @param accessType the access type + * @param userGroups the user groups + * @return if authorized or no + */ + boolean authorize(LensPrivilegeObject lensPrivilegeObject, ActionType accessType, String user, + Set<String> userGroups); +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/DefaultAuthorizer.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/DefaultAuthorizer.java b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/DefaultAuthorizer.java new file mode 100644 index 0000000..553fa3c --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/DefaultAuthorizer.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.authorization; + +import java.util.Set; + +public class DefaultAuthorizer implements Authorizer { + + @Override + public boolean authorize(LensPrivilegeObject lensPrivilegeObject, ActionType accessType, String user, + Set<String> userGroups) { + return true; + } +} http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/LensPrivilegeObject.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/LensPrivilegeObject.java b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/LensPrivilegeObject.java new file mode 100644 index 0000000..afeea95 --- /dev/null +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/authorization/LensPrivilegeObject.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.lens.server.api.authorization; + +import lombok.Data; + +@Data +public class LensPrivilegeObject { + + private final LensPrivilegeObjectType objectType; + private final String table; + private final String column; + + public LensPrivilegeObject(LensPrivilegeObjectType objectType, String table) { + this(objectType, table, null); + } + + public LensPrivilegeObject(LensPrivilegeObjectType objectType, String table, String column) { + this.objectType = objectType; + this.table = table; + this.column = column; + } + + public enum LensPrivilegeObjectType { + DATABASE, COLUMN, NONE + }; + +} + + http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java index 4780955..67038bd 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/metastore/CubeMetastoreService.java @@ -622,5 +622,4 @@ public interface CubeMetastoreService extends LensService, SessionValidator { */ List<String> getAllSegmentations(LensSessionHandle sessionid, String cubeName) throws LensException; - } http://git-wip-us.apache.org/repos/asf/lens/blob/e39dec5f/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java ---------------------------------------------------------------------- diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java index c6a872d..81b415b 100644 --- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java +++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java @@ -19,10 +19,7 @@ package org.apache.lens.server.api.query; import java.io.Serializable; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; +import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -137,8 +134,13 @@ public abstract class AbstractQueryContext implements Serializable { @Setter private Priority priority; + protected AbstractQueryContext(final String query, final String user, final LensConf qconf, final Configuration conf, final Collection<LensDriver> drivers, boolean mergeDriverConf) { + this(query, user, qconf, conf, drivers, mergeDriverConf, null); + } + protected AbstractQueryContext(final String query, final String user, final LensConf qconf, final Configuration conf, + final Collection<LensDriver> drivers, boolean mergeDriverConf, Set<String> userGroups) { if (conf.getBoolean(LensConfConstants.ENABLE_QUERY_METRICS, LensConfConstants.DEFAULT_ENABLE_QUERY_METRICS)) { UUID metricId = UUID.randomUUID(); conf.set(LensConfConstants.QUERY_METRIC_UNIQUE_ID_CONF_KEY, metricId.toString());
