dajac commented on code in PR #15685: URL: https://github.com/apache/kafka/pull/15685#discussion_r1608208322
########## server-common/src/main/java/org/apache/kafka/server/common/FeatureVersion.java: ########## @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.server.common; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public interface FeatureVersion { + + /** + * The level of the feature. 0 means the feature is disabled. + */ + short featureLevel(); + + /** + * The name of the feature. + */ + String featureName(); + + /** + * The next metadata version to be released when the feature became production ready. + * (Ie, if the current production MV is 17 when a feature is released, its mapping should be to MV 18) + */ + MetadataVersion metadataVersionMapping(); Review Comment: I am still confused by this one. Based on our offline discussion, my understanding is that this is only used during bootstrapping. We should try to make this clear in the name and in the javadoc. > (Ie, if the current production MV is 17 when a feature is released, its mapping should be to MV 18) For my understanding, why do we require to be the next one? Requiring the current seems more natural but I may be missing something. ########## server-common/src/main/java/org/apache/kafka/server/common/FeatureVersion.java: ########## @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kafka.server.common; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public interface FeatureVersion { + + /** + * The level of the feature. 0 means the feature is disabled. + */ + short featureLevel(); + + /** + * The name of the feature. + */ + String featureName(); + + /** + * The next metadata version to be released when the feature became production ready. + * (Ie, if the current production MV is 17 when a feature is released, its mapping should be to MV 18) + */ + MetadataVersion metadataVersionMapping(); + + /** + * A mapping from feature to level for all features that this feature depends on. If this feature doesn't + * depend on any others, return an empty map. + * For example, say feature X level x relies on feature Y level y: + * feature (X level x).dependencies() will return (Y -> y) + */ + Map<String, Short> dependencies(); + + /** + * Utility method to map a list of FeatureVersion to a map of feature name to feature level + */ + static Map<String, Short> featureImplsToMap(List<FeatureVersion> features) { + return features.stream().collect(Collectors.toMap(FeatureVersion::featureName, FeatureVersion::featureLevel)); + } Review Comment: This one feels a bit weird in this interface. Should we move it to `Features`? ########## core/src/main/scala/kafka/tools/StorageTool.scala: ########## @@ -109,6 +105,49 @@ object StorageTool extends Logging { } } + private def validateMetadataVersion(metadataVersion: MetadataVersion, config: Option[KafkaConfig]): Unit = { + if (!metadataVersion.isKRaftSupported) { + throw new TerseFailure(s"Must specify a valid KRaft metadata.version of at least ${MetadataVersion.IBP_3_0_IV0}.") + } + if (!metadataVersion.isProduction) { + if (config.get.unstableMetadataVersionsEnabled) { + System.out.println(s"WARNING: using pre-production metadata.version $metadataVersion.") + } else { + throw new TerseFailure(s"The metadata.version $metadataVersion is not ready for production use yet.") + } + } + } + + private[tools] def generateFeatureRecords(metadataRecords: ArrayBuffer[ApiMessageAndVersion], + metadataVersion: MetadataVersion, + specifiedFeatures: Map[String, java.lang.Short], + allFeatures: List[Features]): Unit = { Review Comment: nit: Indentation seems to be off here. ########## server-common/src/main/java/org/apache/kafka/server/common/Features.java: ########## @@ -16,72 +16,135 @@ */ package org.apache.kafka.server.common; -import java.util.Collections; -import java.util.HashMap; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.stream.Collectors; -import static org.apache.kafka.server.common.MetadataVersion.FEATURE_NAME; +/** + * This is enum for the various features implemented for Kafka clusters. + * KIP-584: Versioning Scheme for Features introduced the idea of various features, but only added one feature -- MetadataVersion. + * KIP-1022: Formatting and Updating Features allowed for more features to be added. In order to set and update features, + * they need to be specified via the StorageTool or FeatureCommand tools. + * <br> + * Having a unified enum for the features that will use a shared type in the API used to set and update them + * makes it easier to process these features. + */ +public enum Features { + + /** + * Features defined. If a feature is included in this list, and marked to be used in production they will also be specified when + * formatting a cluster via the StorageTool. MetadataVersion is handled separately, so it is not included here. + * + * See {@link TestFeatureVersion} as an example. See {@link FeatureVersion} when implementing a new feature. + */ + TEST_VERSION("test.feature.version", TestFeatureVersion.values(), TestFeatureVersion::fromFeatureLevel, false); -public final class Features { - private final MetadataVersion version; - private final Map<String, Short> finalizedFeatures; - private final long finalizedFeaturesEpoch; + public static final Features[] FEATURES; + public static final List<Features> PRODUCTION_FEATURES; + private final String name; + private final FeatureVersion[] features; + private final CreateMethod createFeatureVersionMethod; + private final boolean usedInProduction; - public static Features fromKRaftVersion(MetadataVersion version) { - return new Features(version, Collections.emptyMap(), -1, true); + Features(String name, + FeatureVersion[] features, + CreateMethod createMethod, + boolean usedInProduction) { + this.name = name; + this.features = features; + this.createFeatureVersionMethod = createMethod; + this.usedInProduction = usedInProduction; } - public Features( - MetadataVersion version, - Map<String, Short> finalizedFeatures, - long finalizedFeaturesEpoch, - boolean kraftMode - ) { - this.version = version; - this.finalizedFeatures = new HashMap<>(finalizedFeatures); - this.finalizedFeaturesEpoch = finalizedFeaturesEpoch; - // In KRaft mode, we always include the metadata version in the features map. - // In ZK mode, we never include it. - if (kraftMode) { - this.finalizedFeatures.put(FEATURE_NAME, version.featureLevel()); - } else { - this.finalizedFeatures.remove(FEATURE_NAME); - } + static { + Features[] enumValues = Features.values(); + FEATURES = Arrays.copyOf(enumValues, enumValues.length); + + PRODUCTION_FEATURES = Arrays.stream(FEATURES).filter(feature -> + feature.usedInProduction).collect(Collectors.toList()); } - public MetadataVersion metadataVersion() { - return version; + public String featureName() { + return name; } - public Map<String, Short> finalizedFeatures() { - return finalizedFeatures; + public FeatureVersion[] features() { + return features; } - public long finalizedFeaturesEpoch() { - return finalizedFeaturesEpoch; + /** + * Creates a FeatureVersion from a given name and level with the correct feature object underneath. + * + * @param level the level of the feature + * @returns the FeatureVersionUtils.FeatureVersion for the feature the enum is based on. + * @throws IllegalArgumentException if the feature name is not valid (not implemented for this method) + */ + public FeatureVersion fromFeatureLevel(short level) { + return createFeatureVersionMethod.fromFeatureLevel(level); } - @Override - public boolean equals(Object o) { - if (o == null || !(o.getClass().equals(Features.class))) return false; - Features other = (Features) o; - return version == other.version && - finalizedFeatures.equals(other.finalizedFeatures) && - finalizedFeaturesEpoch == other.finalizedFeaturesEpoch; + /** + * A method to validate the feature can be set. If a given feature relies on another feature, the dependencies should be + * captured in {@link FeatureVersion#dependencies()} + * <p> + * For example, say feature X level x relies on feature Y level y: + * if feature X >= x then throw an error if feature Y < y. + * + * All feature levels above 0 require metadata.version=4 (IBP_3_3_IV0) in order to write the feature records to the cluster. + * + * @param feature the feature we are validating + * @param metadataVersion the metadata version we have (or want to set) + * @param features the feature versions (besides MetadataVersion) we have (or want to set) + * @throws IllegalArgumentException if the feature is not valid + */ + public static void validateVersion(FeatureVersion feature, MetadataVersion metadataVersion, Map<String, Short> features) { + if (feature.featureLevel() >= 1 && metadataVersion.isLessThan(MetadataVersion.IBP_3_3_IV0)) + throw new IllegalArgumentException(feature.featureName() + " could not be set to " + feature.featureLevel() + + " because it depends on metadata.version=14 (" + MetadataVersion.IBP_3_3_IV0 + ")"); + + for (Map.Entry<String, Short> dependency: feature.dependencies().entrySet()) { + Short featureLevel = features.get(dependency.getKey()); + + if (featureLevel == null || featureLevel < dependency.getValue()) { + throw new IllegalArgumentException(feature.featureName() + " could not be set to " + feature.featureLevel() + + " because it depends on " + dependency.getKey() + " level " + dependency.getValue()); + } + } } - @Override - public int hashCode() { - return Objects.hash(version, finalizedFeatures, finalizedFeaturesEpoch); + /** + * A method to return the default version level of a feature. If metadataVersionOpt is not empty, the default is based on Review Comment: nit: `metadataVersionOpt` is not there anymore. We should update the javadoc. ########## server-common/src/main/java/org/apache/kafka/server/common/Features.java: ########## @@ -16,72 +16,135 @@ */ package org.apache.kafka.server.common; -import java.util.Collections; -import java.util.HashMap; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import java.util.Map; -import java.util.Objects; +import java.util.stream.Collectors; -import static org.apache.kafka.server.common.MetadataVersion.FEATURE_NAME; +/** + * This is enum for the various features implemented for Kafka clusters. + * KIP-584: Versioning Scheme for Features introduced the idea of various features, but only added one feature -- MetadataVersion. + * KIP-1022: Formatting and Updating Features allowed for more features to be added. In order to set and update features, + * they need to be specified via the StorageTool or FeatureCommand tools. + * <br> + * Having a unified enum for the features that will use a shared type in the API used to set and update them + * makes it easier to process these features. + */ +public enum Features { + + /** + * Features defined. If a feature is included in this list, and marked to be used in production they will also be specified when + * formatting a cluster via the StorageTool. MetadataVersion is handled separately, so it is not included here. + * + * See {@link TestFeatureVersion} as an example. See {@link FeatureVersion} when implementing a new feature. + */ + TEST_VERSION("test.feature.version", TestFeatureVersion.values(), TestFeatureVersion::fromFeatureLevel, false); -public final class Features { - private final MetadataVersion version; - private final Map<String, Short> finalizedFeatures; - private final long finalizedFeaturesEpoch; + public static final Features[] FEATURES; + public static final List<Features> PRODUCTION_FEATURES; + private final String name; + private final FeatureVersion[] features; Review Comment: nit: `versions` or `featureVersions`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
