mimaison commented on code in PR #13278: URL: https://github.com/apache/kafka/pull/13278#discussion_r1268090551
########## tools/src/main/java/org/apache/kafka/tools/CoreUtils.java: ########## @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * General helper functions! + * + * This is for general helper functions that aren't specific to Kafka logic. Things that should have been included in + * the standard library etc. + * + * If you are making a new helper function and want to add it to this class please ensure the following: + * 1. It has documentation + * 2. It is the most general possible utility, not just the thing you needed in one particular place + * 3. You have tests for it if it is nontrivial in any way + */ +public class CoreUtils { Review Comment: Maybe `ToolsUtils` would be a better name? ########## tools/src/test/java/org/apache/kafka/tools/DeleteRecordsCommandTest.java: ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools; + +import kafka.test.ClusterInstance; +import kafka.test.annotation.ClusterTest; +import kafka.test.annotation.ClusterTestDefaults; +import kafka.test.annotation.Type; +import kafka.test.junit.ClusterTestExtensions; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.serialization.StringSerializer; +import org.apache.kafka.server.common.AdminCommandFailedException; +import org.apache.kafka.server.common.AdminOperationException; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@ExtendWith(value = ClusterTestExtensions.class) +@ClusterTestDefaults(clusterType = Type.ALL) +@Tag("integration") +public class DeleteRecordsCommandTest { + + private final ClusterInstance cluster; + public DeleteRecordsCommandTest(ClusterInstance cluster) { + this.cluster = cluster; + } + + @ClusterTest + public void testCommandZk() throws Exception { + Properties adminProps = new Properties(); + + adminProps.put(AdminClientConfig.RETRIES_CONFIG, 1); + + try (Admin admin = cluster.createAdminClient(adminProps)) { + assertThrows( + AdminCommandFailedException.class, + () -> DeleteRecordsCommand.execute0(admin, "{\"partitions\":[" + + "{\"topic\":\"t\", \"partition\":0, \"offset\":1}," + + "{\"topic\":\"t\", \"partition\":0, \"offset\":1}]" + + "}", System.out), + "Offset json file contains duplicate topic partitions: t-0" + ); + + admin.createTopics(Collections.singleton(new NewTopic("t", 1, (short) 1))).all().get(); + + Properties props = new Properties(); + + props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, cluster.bootstrapServers()); + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + + try (KafkaProducer<?, String> producer = new KafkaProducer<>(props)) { + producer.send(new ProducerRecord<>("t", "1")).get(); + producer.send(new ProducerRecord<>("t", "2")).get(); + producer.send(new ProducerRecord<>("t", "3")).get(); + } + + executeAndAssertOutput( + "{\"partitions\":[{\"topic\":\"t\", \"partition\":0, \"offset\":1}]}", + "partition: t-0\tlow_watermark: 1", + admin + ); + + executeAndAssertOutput( + "{\"partitions\":[{\"topic\":\"t\", \"partition\":42, \"offset\":42}]}", + "partition: t-42\terror", + admin + ); + } + } + + private static void executeAndAssertOutput(String json, String expOut, Admin admin) { + String output = + ToolsTestUtils.captureStandardOut(() -> DeleteRecordsCommand.execute0(admin, json, System.out)); + assertTrue(output.contains(expOut)); + } +} + +/** + * Unit test of {@link DeleteRecordsCommand} tool. + */ +class DeleteRecordsCommandUnitTest { + @Test + public void testOffsetFileNotExists() { + assertThrows(IOException.class, () -> DeleteRecordsCommand.main(new String[]{ + "--bootstrap-server", "localhost:9092", + "--offset-json-file", "/not/existing/file" + })); + } + + @Test + public void testCommandConfigNotExists() { + assertThrows(NoSuchFileException.class, () -> DeleteRecordsCommand.main(new String[] { + "--bootstrap-server", "localhost:9092", + "--offset-json-file", "/not/existing/file", + "--command-config", "/another/not/existing/file" + })); + } + + @Test + public void testWrongVersion() { + assertThrowsAdminOperationException("{\"version\":\"string\"}"); + assertThrowsAdminOperationException("{\"version\":2}"); + } + + @Test + public void testWrongPartitions() { + assertThrowsAdminOperationException("{\"version\":1}"); + assertThrowsAdminOperationException("{\"partitions\":2}"); + assertThrowsAdminOperationException("{\"partitions\":{}}"); + assertThrowsAdminOperationException("{\"partitions\":[{}]}"); + assertThrowsAdminOperationException("{\"partitions\":[{\"topic\":\"t\"}]}"); + assertThrowsAdminOperationException("{\"partitions\":[{\"topic\":\"t\", \"partition\": \"\"}]}"); + assertThrowsAdminOperationException("{\"partitions\":[{\"topic\":\"t\", \"partition\": 0}]}"); + assertThrowsAdminOperationException("{\"partitions\":[{\"topic\":\"t\", \"offset\":0}]}"); Review Comment: Should we have a test for it? ########## tools/src/test/java/org/apache/kafka/tools/CoreUtilsTest.java: ########## @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools; + +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.assertIterableEquals; + +public class CoreUtilsTest { + @Test + public void testDuplicates() { + assertIterableEquals( Review Comment: Can we use `assertEquals()` now that `duplicates()` returns a `Set`? ########## tools/src/main/java/org/apache/kafka/tools/DeleteRecordsCommand.java: ########## @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kafka.tools; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import joptsimple.OptionSpec; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.admin.Admin; +import org.apache.kafka.clients.admin.DeleteRecordsResult; +import org.apache.kafka.clients.admin.RecordsToDelete; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.utils.Utils; +import org.apache.kafka.server.common.AdminCommandFailedException; +import org.apache.kafka.server.common.AdminOperationException; +import org.apache.kafka.server.util.CommandDefaultOptions; +import org.apache.kafka.server.util.CommandLineUtils; +import org.apache.kafka.server.util.Json; +import org.apache.kafka.server.util.json.DecodeJson; +import org.apache.kafka.server.util.json.JsonObject; +import org.apache.kafka.server.util.json.JsonValue; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; +import java.util.StringJoiner; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +/** + * A command for delete records of the given partitions down to the specified offset. + */ +public class DeleteRecordsCommand { + private static final int EARLIEST_VERSION = 1; + + private static final DecodeJson.DecodeInteger INT = new DecodeJson.DecodeInteger(); + + private static final DecodeJson.DecodeLong LONG = new DecodeJson.DecodeLong(); + + private static final DecodeJson.DecodeString STRING = new DecodeJson.DecodeString(); + + public static void main(String[] args) throws Exception { + execute(args, System.out); + } + + static Collection<Tuple<TopicPartition, Long>> parseOffsetJsonStringWithoutDedup(String jsonData) throws JsonProcessingException { + JsonValue js = Json.parseFull(jsonData) + .orElseThrow(() -> new AdminOperationException("The input string is not a valid JSON")); + + Optional<JsonValue> version = js.asJsonObject().get("version"); + + return parseJsonData(version.isPresent() ? version.get().to(INT) : EARLIEST_VERSION, js); + } + + private static Collection<Tuple<TopicPartition, Long>> parseJsonData(int version, JsonValue js) throws JsonMappingException { + if (version == 1) { + JsonValue partitions = js.asJsonObject().get("partitions") + .orElseThrow(() -> new AdminOperationException("Missing partitions field")); + + Collection<Tuple<TopicPartition, Long>> res = new ArrayList<>(); + + Iterator<JsonValue> iterator = partitions.asJsonArray().iterator(); + + while (iterator.hasNext()) { + JsonObject partitionJs = iterator.next().asJsonObject(); + + String topic = partitionJs.apply("topic").to(STRING); + int partition = partitionJs.apply("partition").to(INT); + long offset = partitionJs.apply("offset").to(LONG); + + res.add(new Tuple<>(new TopicPartition(topic, partition), offset)); + } + + return res; + } + + throw new AdminOperationException("Not supported version field value " + version); + } + + public static void execute(String[] args, PrintStream out) throws IOException { + DeleteRecordsCommandOptions opts = new DeleteRecordsCommandOptions(args); + + try (Admin adminClient = createAdminClient(opts)) { + execute(adminClient, Utils.readFileAsString(opts.options.valueOf(opts.offsetJsonFileOpt)), out); + } + } + + static void execute(Admin adminClient, String offsetJsonString, PrintStream out) throws JsonProcessingException { + Collection<Tuple<TopicPartition, Long>> offsetSeq = parseOffsetJsonStringWithoutDedup(offsetJsonString); + + Set<TopicPartition> duplicatePartitions = + CoreUtils.duplicates(offsetSeq.stream().map(Tuple::v1).collect(Collectors.toList())); + + if (!duplicatePartitions.isEmpty()) { + StringJoiner duplicates = new StringJoiner(","); + duplicatePartitions.forEach(tp -> duplicates.add(tp.toString())); + throw new AdminCommandFailedException( + String.format("Offset json file contains duplicate topic partitions: %s", duplicates) + ); + } + + Map<TopicPartition, RecordsToDelete> recordsToDelete = offsetSeq.stream() + .map(tuple -> new Tuple<>(tuple.v1, RecordsToDelete.beforeOffset(tuple.v2))) + .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); + + out.println("Executing records delete operation"); + DeleteRecordsResult deleteRecordsResult = adminClient.deleteRecords(recordsToDelete); + out.println("Records delete operation completed:"); + + deleteRecordsResult.lowWatermarks().forEach((tp, partitionResult) -> { + try { + out.printf("partition: %s\tlow_watermark: %s%n", tp, partitionResult.get().lowWatermark()); + } catch (InterruptedException | ExecutionException e) { + out.printf("partition: %s\terror: %s%n", tp, e.getMessage()); + } + }); + } + + private static Admin createAdminClient(DeleteRecordsCommandOptions opts) throws IOException { + Properties props = opts.options.has(opts.commandConfigOpt) + ? Utils.loadProps(opts.options.valueOf(opts.commandConfigOpt)) + : new Properties(); + props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, opts.options.valueOf(opts.bootstrapServerOpt)); + return Admin.create(props); + } + + private static class DeleteRecordsCommandOptions extends CommandDefaultOptions { + private final OptionSpec<String> bootstrapServerOpt; + private final OptionSpec<String> offsetJsonFileOpt; + private final OptionSpec<String> commandConfigOpt; + + public DeleteRecordsCommandOptions(String[] args) { + super(args); + + bootstrapServerOpt = parser.accepts("bootstrap-server", "REQUIRED: The server to connect to.") + .withRequiredArg() + .describedAs("server(s) to use for bootstrapping") + .ofType(String.class); + + offsetJsonFileOpt = parser.accepts("offset-json-file", "REQUIRED: The JSON file with offset per partition. " + + "The format to use is:\n" + + "{\"partitions\":\n [{\"topic\": \"foo\", \"partition\": 1, \"offset\": 1}],\n \"version\":1\n}") + .withRequiredArg() + .describedAs("Offset json file path") + .ofType(String.class); + + commandConfigOpt = parser.accepts("command-config", "A property file containing configs to be passed to Admin Client.") + .withRequiredArg() + .describedAs("command config property file path") + .ofType(String.class); + + options = parser.parse(args); + + CommandLineUtils.maybePrintHelpOrVersion(this, "This tool helps to delete records of the given partitions down to the specified offset."); + + CommandLineUtils.checkRequiredArgs(parser, options, bootstrapServerOpt, offsetJsonFileOpt); + } + } + + public static final class Tuple<V1, V2> { Review Comment: Do we really need this class? Why can't we use something like `Map`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: jira-unsubscr...@kafka.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
