This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch HDDS-3698-upgrade in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/HDDS-3698-upgrade by this push: new 7bbd480 HDDS-3829. Introduce Layout Feature interface in Ozone. (#1322) 7bbd480 is described below commit 7bbd4805423933064c11b39dc91b5abfdd9c6c7a Author: avijayanhwx <14299376+avijayan...@users.noreply.github.com> AuthorDate: Tue Aug 25 17:26:42 2020 -0700 HDDS-3829. Introduce Layout Feature interface in Ozone. (#1322) --- .../upgrade/AbstractLayoutVersionManager.java | 110 +++++++++++++++++++++ .../apache/hadoop/ozone/upgrade/LayoutFeature.java | 33 ++++++- .../hadoop/ozone/upgrade/LayoutVersionManager.java | 64 ++++++++++++ .../apache/hadoop/ozone/upgrade}/package-info.java | 12 ++- .../upgrade/TestAbstractLayoutVersionManager.java | 85 ++++++++++++++++ hadoop-ozone/ozone-manager/pom.xml | 31 ++++++ .../org/apache/hadoop/ozone/om/OzoneManager.java | 4 + .../om/response/s3/security/package-info.java | 2 +- .../NewOmFeatureUpgradeAction.java} | 20 +++- .../OMLayoutFeatureAPI.java} | 23 ++++- .../ozone/om/upgrade/OMLayoutFeatureAspect.java | 54 ++++++++++ .../ozone/om/upgrade/OMLayoutFeatureCatalog.java | 93 +++++++++++++++++ .../ozone/om/upgrade/OMLayoutVersionManager.java | 92 +++++++++++++++++ .../OmUpgradeAction.java} | 16 ++- .../s3/security => upgrade}/package-info.java | 7 +- .../request/volume/TestOMVolumeDeleteRequest.java | 2 +- .../om/upgrade/TestOMLayoutFeatureAspect.java | 71 +++++++++++++ .../ozone/om/upgrade/TestOMVersionManager.java | 66 +++++++++++++ 18 files changed, 755 insertions(+), 30 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java new file mode 100644 index 0000000..99f72c8 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/AbstractLayoutVersionManager.java @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +import com.google.common.base.Preconditions; + +/** + * Layout Version Manager containing generic method implementations. + */ +@SuppressWarnings("visibilitymodifier") +public abstract class AbstractLayoutVersionManager implements + LayoutVersionManager { + + protected int metadataLayoutVersion; // MLV. + protected int softwareLayoutVersion; // SLV. + protected TreeMap<Integer, LayoutFeature> features = new TreeMap<>(); + protected Map<String, LayoutFeature> featureMap = new HashMap<>(); + protected volatile boolean isInitialized = false; + + protected void init(int version, LayoutFeature[] lfs) { + if (!isInitialized) { + metadataLayoutVersion = version; + initializeFeatures(lfs); + softwareLayoutVersion = features.lastKey(); + isInitialized = true; + } + } + + protected void initializeFeatures(LayoutFeature[] lfs) { + Arrays.stream(lfs).forEach(f -> { + Preconditions.checkArgument(!featureMap.containsKey(f.name())); + Preconditions.checkArgument(!features.containsKey(f.layoutVersion())); + features.put(f.layoutVersion(), f); + featureMap.put(f.name(), f); + }); + } + + public int getMetadataLayoutVersion() { + return metadataLayoutVersion; + } + + public int getSoftwareLayoutVersion() { + return softwareLayoutVersion; + } + + public boolean needsFinalization() { + return metadataLayoutVersion < softwareLayoutVersion; + } + + public boolean isAllowed(LayoutFeature layoutFeature) { + return layoutFeature.layoutVersion() <= metadataLayoutVersion; + } + + public boolean isAllowed(String featureName) { + return featureMap.containsKey(featureName) && + isAllowed(featureMap.get(featureName)); + } + + public LayoutFeature getFeature(String name) { + return featureMap.get(name); + } + + public void doFinalize(Object param) { + if (needsFinalization()){ + Iterator<Map.Entry<Integer, LayoutFeature>> iterator = features + .tailMap(metadataLayoutVersion + 1).entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry<Integer, LayoutFeature> f = iterator.next(); + Optional<? extends LayoutFeature.UpgradeAction> upgradeAction = + f.getValue().onFinalizeAction(); + upgradeAction.ifPresent(action -> action.executeAction(param)); + // ToDo : Handle shutdown while iterating case (resume from last + // feature). + metadataLayoutVersion = f.getKey(); + } + // ToDo : Persist new MLV. + } + } + + protected void reset() { + metadataLayoutVersion = 0; + softwareLayoutVersion = 0; + featureMap.clear(); + features.clear(); + isInitialized = false; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java similarity index 55% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java copy to hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java index d9024d1..05e944e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutFeature.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,30 @@ * limitations under the License. */ +package org.apache.hadoop.ozone.upgrade; + +import java.util.Optional; + /** - * Package contains classes related to S3 security responses. + * Generic Layout feature interface for Ozone. */ -package org.apache.hadoop.ozone.om.request.s3.security; +public interface LayoutFeature { + String name(); + + int layoutVersion(); + + String description(); + + default Optional<? extends UpgradeAction> onFinalizeAction() { + return Optional.empty(); + } + + /** + * Generic UpgradeAction interface. An operation that is run on specific + * upgrade states like post finalize, pre-downgrade etc. + * @param <T> + */ + interface UpgradeAction<T> { + void executeAction(T arg); + } +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManager.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManager.java new file mode 100644 index 0000000..432bd52 --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/LayoutVersionManager.java @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +/** + * Read Only interface to an Ozone component's Version Manager. + */ +public interface LayoutVersionManager { + + /** + * Get the Current Metadata Layout Version. + * @return MLV + */ + int getMetadataLayoutVersion(); + + /** + * Get the Current Software Layout Version. + * @return SLV + */ + int getSoftwareLayoutVersion(); + + /** + * Does it need finalization? + * @return true/false + */ + boolean needsFinalization(); + + /** + * Is allowed feature? + * @param layoutFeature feature object + * @return true/false. + */ + boolean isAllowed(LayoutFeature layoutFeature); + + /** + * Is allowed feature? + * @param featureName feature name + * @return true/false. + */ + boolean isAllowed(String featureName); + + /** + * Get Feature given feature name. + * @param name Feature name. + * @return LayoutFeature instance. + */ + LayoutFeature getFeature(String name); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java similarity index 80% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java copy to hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java index d9024d1..cf992d8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/upgrade/package-info.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,6 +17,8 @@ */ /** - * Package contains classes related to S3 security responses. + * This package contains classes for the Ozone upgrade and layout version + * management. */ -package org.apache.hadoop.ozone.om.request.s3.security; +package org.apache.hadoop.ozone.upgrade; + diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java new file mode 100644 index 0000000..44fa100 --- /dev/null +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/ozone/upgrade/TestAbstractLayoutVersionManager.java @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.upgrade; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Before; +import org.junit.Test; + +/** + * Test generic layout management init and APIs. + */ +public class TestAbstractLayoutVersionManager { + + private AbstractLayoutVersionManager versionManager = + new MockVersionManager(); + + @Before + public void setUp() { + versionManager.reset(); + } + + @Test + public void testInit() { + versionManager.init(1, + getTestLayoutFeatures(2)); + assertEquals(2, versionManager.features.size()); + assertEquals(2, versionManager.featureMap.size()); + assertEquals(1, versionManager.getMetadataLayoutVersion()); + assertEquals(2, versionManager.getSoftwareLayoutVersion()); + assertTrue(versionManager.needsFinalization()); + } + + @Test + public void testNeedsFinalization() { + versionManager.init(2, getTestLayoutFeatures(2)); + assertFalse(versionManager.needsFinalization()); + } + + private LayoutFeature[] getTestLayoutFeatures(int num) { + LayoutFeature[] lfs = new LayoutFeature[num]; + int k = 0; + for (int i = 1; i <= num; i++) { + int finalI = i; + lfs[k++] = new LayoutFeature() { + @Override + public String name() { + return "LF-" + finalI; + } + + @Override + public int layoutVersion() { + return finalI; + } + + @Override + public String description() { + return null; + } + }; + } + return lfs; + } + + static class MockVersionManager extends AbstractLayoutVersionManager { + } +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml index 056498a..f1bbd97 100644 --- a/hadoop-ozone/ozone-manager/pom.xml +++ b/hadoop-ozone/ozone-manager/pom.xml @@ -31,6 +31,18 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> <dependencies> <dependency> + <groupId>org.aspectj</groupId> + <artifactId>aspectjrt</artifactId> + <version>1.8.9</version> + </dependency> + + <dependency> + <groupId>org.aspectj</groupId> + <artifactId>aspectjweaver</artifactId> + <version>1.8.9</version> + </dependency> + + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-ozone-common</artifactId> </dependency> @@ -145,6 +157,25 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> </execution> </executions> </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>aspectj-maven-plugin</artifactId> + <version>1.10</version> + <configuration> + <source>1.8</source> + <target>1.8</target> + </configuration> + <executions> + <execution> + <goals> + <goal>compile</goal> + </goals> + <configuration> + <complianceLevel>1.8</complianceLevel> + </configuration> + </execution> + </executions> + </plugin> </plugins> <testResources> <testResource> diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 13e47a7..6f368e1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -140,6 +140,7 @@ import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.snapshot.OzoneManagerSnapshotProvider; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; @@ -1140,6 +1141,9 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl metadataManager.start(configuration); startSecretManagerIfNecessary(); + OMLayoutVersionManager omVersionManager = + OMLayoutVersionManager.initialize(omStorage); + if (certClient != null) { caCertPem = CertificateCodec.getPEMEncodedString( certClient.getCACertificate()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java index d9024d1..eb86413 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java @@ -19,4 +19,4 @@ /** * Package contains classes related to S3 security responses. */ -package org.apache.hadoop.ozone.om.request.s3.security; +package org.apache.hadoop.ozone.om.response.s3.security; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/NewOmFeatureUpgradeAction.java similarity index 66% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/NewOmFeatureUpgradeAction.java index d9024d1..bb4eb7c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/NewOmFeatureUpgradeAction.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,17 @@ * limitations under the License. */ +package org.apache.hadoop.ozone.om.upgrade; + +import org.apache.hadoop.ozone.om.OzoneManager; + /** - * Package contains classes related to S3 security responses. + * Stub OM Action class to help with understanding. Will be removed. */ -package org.apache.hadoop.ozone.om.request.s3.security; +public class NewOmFeatureUpgradeAction implements OmUpgradeAction { + + @Override + public void executeAction(OzoneManager ozoneManager) { + // Do blah.... + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAPI.java similarity index 58% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAPI.java index d9024d1..2da8b38 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAPI.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,20 @@ * limitations under the License. */ +package org.apache.hadoop.ozone.om.upgrade; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeatureCatalog.OMLayoutFeature; + /** - * Package contains classes related to S3 security responses. + * Annotation to specify if an API is backed up by a Layout Feature. */ -package org.apache.hadoop.ozone.om.request.s3.security; +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface OMLayoutFeatureAPI { + OMLayoutFeature value(); +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java new file mode 100644 index 0000000..a92e3b4 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureAspect.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION; + +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.upgrade.LayoutFeature; +import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; +import org.aspectj.lang.reflect.MethodSignature; + +/** + * 'Aspect' for OM Layout Feature API. All methods annotated with the + * specific annotation will have pre-processing done here to check layout + * version compatibility. + */ +@Aspect +public class OMLayoutFeatureAspect { + + @Before("@annotation(OMLayoutFeatureAPI) && execution(* *(..))") + public void checkLayoutFeature(JoinPoint joinPoint) throws Throwable { + String featureName = ((MethodSignature) joinPoint.getSignature()) + .getMethod().getAnnotation(OMLayoutFeatureAPI.class).value().name(); + LayoutVersionManager lvm = OMLayoutVersionManager.getInstance(); + if (!lvm.isAllowed(featureName)) { + LayoutFeature layoutFeature = lvm.getFeature(featureName); + throw new OMException(String.format("Operation %s cannot be invoked " + + "before finalization. Current layout version = %d, feature's layout" + + " version = %d", + featureName, + lvm.getMetadataLayoutVersion(), + layoutFeature.layoutVersion()), NOT_SUPPORTED_OPERATION); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureCatalog.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureCatalog.java new file mode 100644 index 0000000..c5ed27d --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeatureCatalog.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import java.util.Optional; + +import org.apache.hadoop.ozone.upgrade.LayoutFeature; + +/** + * Catalog of Ozone Manager features. + */ +public class OMLayoutFeatureCatalog { + + /** + * List of OM Features. + */ + public enum OMLayoutFeature implements LayoutFeature { + INITIAL_VERSION(0, "Initial Layout Version"), + CREATE_EC(1, ""), + NEW_FEATURE(2, "new feature", new NewOmFeatureUpgradeAction()); + + + private int layoutVersion; + private String description; + private Optional<OmUpgradeAction> omUpgradeAction = Optional.empty(); + + OMLayoutFeature(final int layoutVersion, String description) { + this.layoutVersion = layoutVersion; + this.description = description; + } + + OMLayoutFeature(final int layoutVersion, String description, + OmUpgradeAction upgradeAction) { + this.layoutVersion = layoutVersion; + this.description = description; + omUpgradeAction = Optional.of(upgradeAction); + } + + @Override + public int layoutVersion() { + return layoutVersion; + } + + @Override + public String description() { + return description; + } + + @Override + public Optional<OmUpgradeAction> onFinalizeAction() { + return omUpgradeAction; + } + } + + /** + * This is an example of an "API" that uses a new Layout feature (EC) that is + * not yet supported by the current layout version. The following can be + * "guarded" by just adding the following annotation, thereby keeping the + * method logic and upgrade logic separate. + */ + @OMLayoutFeatureAPI(OMLayoutFeature.CREATE_EC) + public String ecMethod() { + // Blah Blah EC Blah.... + return "ec"; + } + + /** + * This is an example of an "API" that uses a Layout feature (EC) that is + * supported by the current layout version. + */ + @OMLayoutFeatureAPI(OMLayoutFeature.INITIAL_VERSION) + public String basicMethod() { + // Blah Blah Basic Blah.... + return "basic"; + } +} + diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java new file mode 100644 index 0000000..2f959a9 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutVersionManager.java @@ -0,0 +1,92 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION; + +import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeatureCatalog.OMLayoutFeature; +import org.apache.hadoop.ozone.upgrade.AbstractLayoutVersionManager; +import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Class to manage layout versions and features for Ozone Manager. + */ +public final class OMLayoutVersionManager extends AbstractLayoutVersionManager { + + private static OMLayoutVersionManager omVersionManager; + + private OMLayoutVersionManager() { + } + + /** + * Read only instance to OM Version Manager. + * @return version manager instance. + */ + public static synchronized LayoutVersionManager getInstance() { + if (omVersionManager == null) { + throw new RuntimeException("OM Layout Version Manager not yet " + + "initialized."); + } + return omVersionManager; + } + + + /** + * Initialize OM version manager from storage. + * @return version manager instance. + */ + public static synchronized OMLayoutVersionManager initialize( + OMStorage omStorage) + throws OMException { + if (omVersionManager == null) { + omVersionManager = new OMLayoutVersionManager(); + omVersionManager.init(omStorage); + } + return omVersionManager; + } + + /** + * Initialize the OM Layout Features and current Layout Version. + * @param storage to read the current layout version. + * @throws OMException on error. + */ + private void init(Storage storage) throws OMException { + init(storage.getLayoutVersion(), OMLayoutFeature.values()); + if (metadataLayoutVersion > softwareLayoutVersion) { + throw new OMException( + String.format("Cannot initialize VersionManager. Metadata " + + "layout version (%d) > software layout version (%d)", + metadataLayoutVersion, softwareLayoutVersion), + NOT_SUPPORTED_OPERATION); + } + } + + @VisibleForTesting + protected synchronized static void resetLayoutVersionManager() { + if (omVersionManager != null) { + omVersionManager.reset(); + omVersionManager = null; + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OmUpgradeAction.java similarity index 67% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OmUpgradeAction.java index d9024d1..da9a063 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OmUpgradeAction.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,7 +16,13 @@ * limitations under the License. */ +package org.apache.hadoop.ozone.om.upgrade; + +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.upgrade.LayoutFeature.UpgradeAction; + /** - * Package contains classes related to S3 security responses. + * Upgrade Action for OzoneManager which takes in an 'OM' instance. */ -package org.apache.hadoop.ozone.om.request.s3.security; +public interface OmUpgradeAction extends UpgradeAction<OzoneManager> { +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/package-info.java similarity index 86% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/package-info.java index d9024d1..d663049 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/s3/security/package-info.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/package-info.java @@ -16,7 +16,8 @@ * limitations under the License. */ +package org.apache.hadoop.ozone.om.upgrade; + /** - * Package contains classes related to S3 security responses. - */ -package org.apache.hadoop.ozone.om.request.s3.security; + * This package contains OM Upgrade related classes. + */ \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeDeleteRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeDeleteRequest.java index 49f28d3..73a2888 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/volume/TestOMVolumeDeleteRequest.java @@ -20,7 +20,7 @@ package org.apache.hadoop.ozone.om.request.volume; import java.util.UUID; -import org.junit.Assert;; +import org.junit.Assert; import org.junit.Test; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java new file mode 100644 index 0000000..b68c7c2 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMLayoutFeatureAspect.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NOT_SUPPORTED_OPERATION; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** + * Class to test annotation based interceptor that checks whether layout + * feature API is allowed. + */ +public class TestOMLayoutFeatureAspect { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private OzoneConfiguration configuration = new OzoneConfiguration(); + + @Before + public void setUp() throws IOException { + configuration.set("ozone.metadata.dirs", + temporaryFolder.newFolder().getAbsolutePath()); + } + + /** + * This unit test invokes the above 2 layout feature APIs. The first one + * should fail, and the second one should pass. + * @throws Exception + */ + @Test + public void testCheckLayoutFeature() throws Exception { + OMLayoutVersionManager.initialize(new OMStorage(configuration)); + OMLayoutFeatureCatalog testObj = new OMLayoutFeatureCatalog(); + try { + testObj.ecMethod(); + Assert.fail(); + } catch (Exception ex) { + OMException omEx = (OMException) ex; + assertEquals(NOT_SUPPORTED_OPERATION, omEx.getResult()); + } + String s = testObj.basicMethod(); + assertEquals("basic", s); + } +} \ No newline at end of file diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java new file mode 100644 index 0000000..cfcfe24 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/upgrade/TestOMVersionManager.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.upgrade; + +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeatureCatalog.OMLayoutFeature.CREATE_EC; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutFeatureCatalog.OMLayoutFeature.INITIAL_VERSION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.apache.hadoop.ozone.om.OMStorage; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeatureCatalog.OMLayoutFeature; +import org.apache.hadoop.ozone.upgrade.LayoutFeature; +import org.junit.Test; + +/** + * Test OM layout version management. + */ +public class TestOMVersionManager { + + @Test + public void testOMLayoutVersionManager() throws IOException { + OMStorage omStorage = mock(OMStorage.class); + when(omStorage.getLayoutVersion()).thenReturn(0); + OMLayoutVersionManager omVersionManager = + OMLayoutVersionManager.initialize(omStorage); + assertTrue(omVersionManager.isAllowed(INITIAL_VERSION)); + assertFalse(omVersionManager.isAllowed(CREATE_EC)); + assertEquals(0, omVersionManager.getMetadataLayoutVersion()); + assertTrue(omVersionManager.needsFinalization()); + omVersionManager.doFinalize(mock(OzoneManager.class)); + assertFalse(omVersionManager.needsFinalization()); + assertEquals(2, omVersionManager.getMetadataLayoutVersion()); + } + + @Test + public void testOMLayoutFeatureCatalog() { + OMLayoutFeature[] values = OMLayoutFeature.values(); + int currVersion = Integer.MIN_VALUE; + for (LayoutFeature lf : values) { + assertTrue(currVersion <= lf.layoutVersion()); + currVersion = lf.layoutVersion(); + } + } +} \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org