This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch 1.9 in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push: new 2d0091d Add optional -t tablename to importdirectory shell command. (#1299) 2d0091d is described below commit 2d0091d7136c216e1e043d462c34870f4e68ba55 Author: EdColeman <d...@etcoleman.com> AuthorDate: Thu Aug 15 15:41:15 2019 -0400 Add optional -t tablename to importdirectory shell command. (#1299) * Add optional -t tablename to importdirectory shell command. the -t tablename is optional to preserve original behaviour. Without the option, the current table is used. * added getOptions * wip - tests cmd format, need to evaluate testing functionality --- .../shell/commands/ImportDirectoryCommand.java | 25 +++- .../shell/commands/ImportDirectoryCommandTest.java | 130 +++++++++++++++++++++ .../org/apache/accumulo/test/ShellServerIT.java | 62 ++++++++++ 3 files changed, 211 insertions(+), 6 deletions(-) diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java index ec1576b..dc92293 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java @@ -24,37 +24,50 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.shell.Shell; import org.apache.accumulo.shell.Shell.Command; import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; public class ImportDirectoryCommand extends Command { @Override public String description() { - return "bulk imports an entire directory of data files to the current" - + " table. The boolean argument determines if accumulo sets the time."; + return "bulk imports an entire directory of data files into an existing table." + + " The table is either passed with the -t tablename opt, or into to the current" + + " table if the -t option is not provided. The boolean argument determines if accumulo sets the time."; } @Override public int execute(final String fullCommand, final CommandLine cl, final Shell shellState) throws IOException, AccumuloException, AccumuloSecurityException, TableNotFoundException { - shellState.checkTableState(); + + final String tableName = OptUtil.getTableOpt(cl, shellState); String dir = cl.getArgs()[0]; String failureDir = cl.getArgs()[1]; + final boolean setTime = Boolean.parseBoolean(cl.getArgs()[2]); - shellState.getConnector().tableOperations().importDirectory(shellState.getTableName(), dir, - failureDir, setTime); + shellState.getConnector().tableOperations().importDirectory(tableName, dir, failureDir, + setTime); + return 0; } @Override public int numArgs() { + // arg count for args not handled with Options return 3; } @Override public String usage() { - return getName() + " <directory> <failureDirectory> true|false"; + return getName() + "[-t tablename] <directory> <failureDirectory> true|false"; + } + + @Override + public Options getOptions() { + final Options opts = super.getOptions(); + opts.addOption(OptUtil.tableOpt("name of the table to import files into")); + return opts; } } diff --git a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java new file mode 100644 index 0000000..c819d6c --- /dev/null +++ b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.shell.commands; + +import static org.easymock.EasyMock.expectLastCall; + +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.shell.Shell; +import org.apache.commons.cli.CommandLine; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; + +public class ImportDirectoryCommandTest { + + private ImportDirectoryCommand cmd; + + private Connector conn; + private CommandLine cli; + private Shell shellState; + private TableOperations tableOperations; + + @Before + public void setup() { + cmd = new ImportDirectoryCommand(); + + // Initialize that internal state + cmd.getOptions(); + + conn = EasyMock.createMock(Connector.class); + cli = EasyMock.createMock(CommandLine.class); + shellState = EasyMock.createMock(Shell.class); + tableOperations = EasyMock.createMock(TableOperations.class); + + } + + /** + * Test original command form - no -t tablename option provided. + * + * @throws Exception + * any exception is a test failure. + */ + @Test + public void testOriginalCmdForm() throws Exception { + + String[] cliArgs = {"in_dir", "fail_dir", "false"}; + // + // EasyMock.expect(cli.hasOption('t')).andReturn(false); + + EasyMock.expect(cli.hasOption("t")).andReturn(false); + + EasyMock.expect(cli.getArgs()).andReturn(cliArgs); + EasyMock.expect(cli.getArgs()).andReturn(cliArgs); + EasyMock.expect(cli.getArgs()).andReturn(cliArgs); + + EasyMock.expect(shellState.getConnector()).andReturn(conn); + EasyMock.expect(shellState.getTableName()).andReturn("tablename"); + + shellState.checkTableState(); + expectLastCall().andVoid(); + + // Table exists + EasyMock.expect(conn.tableOperations()).andReturn(tableOperations); + + tableOperations.importDirectory("tablename", "in_dir", "fail_dir", false); + expectLastCall().times(3); + + EasyMock.replay(conn, cli, shellState, tableOperations); + + cmd.execute("importdirectory in_dir fail_dir false", cli, shellState); + + } + + /** + * Test with -t tablename option provided. + * + * @throws Exception + * any exception is a test failure. + */ + @Test + public void testPassTableOptCmdForm() throws Exception { + + String[] cliArgs = {"in_dir", "fail_dir", "false"}; + // + // EasyMock.expect(cli.hasOption('t')).andReturn(false); + + EasyMock.expect(cli.hasOption("t")).andReturn(true); + EasyMock.expect(cli.hasOption("t")).andReturn(true); + EasyMock.expect(cli.getOptionValue("t")).andReturn("passedName"); + + EasyMock.expect(tableOperations.exists("passedName")).andReturn(true); + EasyMock.expect(shellState.getConnector()).andReturn(conn); + EasyMock.expect(conn.tableOperations()).andReturn(tableOperations); + + EasyMock.expect(cli.getArgs()).andReturn(cliArgs); + EasyMock.expect(cli.getArgs()).andReturn(cliArgs); + EasyMock.expect(cli.getArgs()).andReturn(cliArgs); + + EasyMock.expect(shellState.getConnector()).andReturn(conn); + + shellState.checkTableState(); + expectLastCall().andVoid(); + + // Table exists + EasyMock.expect(conn.tableOperations()).andReturn(tableOperations); + + tableOperations.importDirectory("passedName", "in_dir", "fail_dir", false); + expectLastCall().times(3); + + EasyMock.replay(conn, cli, shellState, tableOperations); + + cmd.execute("importdirectory in_dir fail_dir false", cli, shellState); + + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java index 23ef3bf..e429898 100644 --- a/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ShellServerIT.java @@ -1898,6 +1898,68 @@ public class ShellServerIT extends SharedMiniClusterBase { ts.exec("deletetable -f " + table); } + + /** + * Validate importdirectory command accepts addinig -t tablename option or the accepts original + * format that uses the current working table. Currently this test does not validate the actual + * import - only the command syntax. + * + * @throws Exception + * any exception is a test failure. + */ + @Test + public void importDirectoryCmdFmt() throws Exception { + + final String table = name.getMethodName(); + + Configuration conf = new Configuration(); + FileSystem fs = FileSystem.get(conf); + File importDir = new File(rootPath, "import_" + table); + assertTrue(importDir.mkdir()); + + File errorsDir = new File(rootPath, "errors_" + table); + assertTrue(errorsDir.mkdir()); + + // expect fail - table does not exist. + ts.exec(String.format("importdirectory -t %s %s %s false", table, importDir, errorsDir), false); + assertTrue(checkErrorMsg("TableNotFoundException", ts.output.get().split("\n"))); + + ts.exec(String.format("table %s", table), false); + assertTrue(checkErrorMsg("TableNotFoundException", ts.output.get().split("\n"))); + + ts.exec("createtable " + table, true); + + // validate -t option is used. + ts.exec(String.format("importdirectory -t %s %s %s false", table, importDir, errorsDir), true); + + // validate original cmd format. + ts.exec(String.format("table %s", table), true); + ts.exec(String.format("importdirectory %s %s false", importDir, errorsDir), true); + + // expect fail - invalid command, + ts.exec(String.format("importdirectory false"), false); + assertTrue(checkErrorMsg("Expected 3 arguments. There was 1.", ts.output.get().split("\n"))); + + // expect fail - original cmd without a table. + ts.exec("notable", true); + ts.exec(String.format("importdirectory %s %s false", importDir, errorsDir), false); + + assertTrue(checkErrorMsg("java.lang.IllegalStateException: Not in a table context.", + ts.output.get().split("\n"))); + } + + private boolean checkErrorMsg(final String expectedText, final String[] lines) { + boolean foundText = false; + + for (String line : lines) { + if (line.contains(expectedText)) { + foundText = true; + } + log.trace("shell output>: \'{}\'", line); + } + return foundText; + } + private static final String FAKE_CONTEXT = "FAKE"; private static final String FAKE_CONTEXT_CLASSPATH = "file://" + System.getProperty("user.dir") + "/target/" + ShellServerIT.class.getSimpleName() + "-fake-iterators.jar";