[accumulo] branch 1.9 updated: Improve importDirectory tests from #1299
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push:
new 19bdeb4 Improve importDirectory tests from #1299
19bdeb4 is described below
commit 19bdeb4d908b5eece1ebccb6be1295e8cf0759de
Author: Christopher Tubbs
AuthorDate: Thu Aug 15 20:29:18 2019 -0400
Improve importDirectory tests from #1299
* Remove unused variables in test
* Verify mock objects in test
---
.../shell/commands/ImportDirectoryCommandTest.java | 79 +-
.../org/apache/accumulo/test/ShellServerIT.java| 5 --
2 files changed, 32 insertions(+), 52 deletions(-)
diff --git
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
index c819d6c..06694fd 100644
---
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
@@ -16,13 +16,17 @@
*/
package org.apache.accumulo.shell.commands;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.shell.Shell;
import org.apache.commons.cli.CommandLine;
-import org.easymock.EasyMock;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -42,11 +46,15 @@ public class ImportDirectoryCommandTest {
// Initialize that internal state
cmd.getOptions();
-conn = EasyMock.createMock(Connector.class);
-cli = EasyMock.createMock(CommandLine.class);
-shellState = EasyMock.createMock(Shell.class);
-tableOperations = EasyMock.createMock(TableOperations.class);
+conn = createMock(Connector.class);
+cli = createMock(CommandLine.class);
+shellState = createMock(Shell.class);
+tableOperations = createMock(TableOperations.class);
+ }
+ @After
+ public void verifyMocks() {
+verify(conn, cli, shellState, tableOperations);
}
/**
@@ -57,33 +65,24 @@ public class ImportDirectoryCommandTest {
*/
@Test
public void testOriginalCmdForm() throws Exception {
-
String[] cliArgs = {"in_dir", "fail_dir", "false"};
-//
-// EasyMock.expect(cli.hasOption('t')).andReturn(false);
-EasyMock.expect(cli.hasOption("t")).andReturn(false);
+// no -t option, use current table context
+expect(cli.hasOption("t")).andReturn(false).once();
+expect(shellState.getTableName()).andReturn("tablename").once();
-EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
-EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
-EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
-
-EasyMock.expect(shellState.getConnector()).andReturn(conn);
-EasyMock.expect(shellState.getTableName()).andReturn("tablename");
+expect(cli.getArgs()).andReturn(cliArgs).atLeastOnce();
+expect(shellState.getConnector()).andReturn(conn).atLeastOnce();
+expect(conn.tableOperations()).andReturn(tableOperations);
shellState.checkTableState();
-expectLastCall().andVoid();
-
-// Table exists
-EasyMock.expect(conn.tableOperations()).andReturn(tableOperations);
+expectLastCall().once();
tableOperations.importDirectory("tablename", "in_dir", "fail_dir", false);
-expectLastCall().times(3);
-
-EasyMock.replay(conn, cli, shellState, tableOperations);
+expectLastCall().once();
+replay(conn, cli, shellState, tableOperations);
cmd.execute("importdirectory in_dir fail_dir false", cli, shellState);
-
}
/**
@@ -94,37 +93,23 @@ public class ImportDirectoryCommandTest {
*/
@Test
public void testPassTableOptCmdForm() throws Exception {
-
String[] cliArgs = {"in_dir", "fail_dir", "false"};
-//
-// EasyMock.expect(cli.hasOption('t')).andReturn(false);
-EasyMock.expect(cli.hasOption("t")).andReturn(true);
-EasyMock.expect(cli.hasOption("t")).andReturn(true);
-EasyMock.expect(cli.getOptionValue("t")).andReturn("passedName");
+// -t option specified, table is from option
+expect(cli.hasOption("t")).andReturn(true).once();
+expect(cli.getOptionValue("t")).andReturn("passedName").once();
+expect(tableOperations.exists("passedName")).andReturn(true).once();
-EasyMock.expect(tableOperations.exists("passedName")).andReturn(true);
-EasyMock.expect(shellState.getConnector()).andReturn(conn);
-EasyMock.expect(conn.tableOperations()).andReturn(tableOperations);
+expect(cli.getArgs()).andReturn(cliArgs).atLeastOnce();
+
[accumulo] branch master updated (43a7629 -> 7949aae)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 43a7629 Merge branch '2.0' new 19bdeb4 Improve importDirectory tests from #1299 new 4e67b51 Merge branch '1.9' into 2.0 new 6aa4c5f Use newer importDirectory method for unit test of #1299 new 7949aae Merge branch '2.0' The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../shell/commands/ImportDirectoryCommandTest.java | 100 ++--- .../org/apache/accumulo/test/ShellServerIT.java| 4 - 2 files changed, 47 insertions(+), 57 deletions(-)
[accumulo] branch 2.0 updated (a6b72c5 -> 6aa4c5f)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from a6b72c5 Merge 1.9 into 2.0 new 19bdeb4 Improve importDirectory tests from #1299 new 4e67b51 Merge branch '1.9' into 2.0 new 6aa4c5f Use newer importDirectory method for unit test of #1299 The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../shell/commands/ImportDirectoryCommandTest.java | 100 ++--- .../org/apache/accumulo/test/ShellServerIT.java| 4 - 2 files changed, 47 insertions(+), 57 deletions(-)
[accumulo] 01/01: Merge branch '2.0'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 7949aae21355e8f3fd25541a894344dbb1db57b9 Merge: 43a7629 6aa4c5f Author: Christopher Tubbs AuthorDate: Thu Aug 15 23:22:54 2019 -0400 Merge branch '2.0' .../shell/commands/ImportDirectoryCommandTest.java | 100 ++--- .../org/apache/accumulo/test/ShellServerIT.java| 4 - 2 files changed, 47 insertions(+), 57 deletions(-)
[accumulo] 01/02: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 4e67b51dcadedecfdf70ab7f58cfffc10682513f
Merge: a6b72c5 19bdeb4
Author: Christopher Tubbs
AuthorDate: Thu Aug 15 22:31:45 2019 -0400
Merge branch '1.9' into 2.0
.../shell/commands/ImportDirectoryCommandTest.java | 79 +-
.../org/apache/accumulo/test/ShellServerIT.java| 4 --
2 files changed, 32 insertions(+), 51 deletions(-)
diff --cc
shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
index 963d919,06694fd..556ad5a
---
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
@@@ -16,9 -16,13 +16,13 @@@
*/
package org.apache.accumulo.shell.commands;
+ import static org.easymock.EasyMock.createMock;
+ import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
+ import static org.easymock.EasyMock.replay;
+ import static org.easymock.EasyMock.verify;
-import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.shell.Shell;
import org.apache.commons.cli.CommandLine;
@@@ -42,11 -46,15 +46,15 @@@ public class ImportDirectoryCommandTes
// Initialize that internal state
cmd.getOptions();
- client = EasyMock.createMock(AccumuloClient.class);
- cli = EasyMock.createMock(CommandLine.class);
- shellState = EasyMock.createMock(Shell.class);
- tableOperations = EasyMock.createMock(TableOperations.class);
-conn = createMock(Connector.class);
++client = createMock(AccumuloClient.class);
+ cli = createMock(CommandLine.class);
+ shellState = createMock(Shell.class);
+ tableOperations = createMock(TableOperations.class);
+ }
+ @After
+ public void verifyMocks() {
-verify(conn, cli, shellState, tableOperations);
++verify(client, cli, shellState, tableOperations);
}
/**
@@@ -57,33 -65,24 +65,24 @@@
*/
@Test
public void testOriginalCmdForm() throws Exception {
-
String[] cliArgs = {"in_dir", "fail_dir", "false"};
- //
- // EasyMock.expect(cli.hasOption('t')).andReturn(false);
- EasyMock.expect(cli.hasOption("t")).andReturn(false);
+ // no -t option, use current table context
+ expect(cli.hasOption("t")).andReturn(false).once();
+ expect(shellState.getTableName()).andReturn("tablename").once();
- EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
- EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
- EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
-
- EasyMock.expect(shellState.getAccumuloClient()).andReturn(client);
- EasyMock.expect(shellState.getTableName()).andReturn("tablename");
+ expect(cli.getArgs()).andReturn(cliArgs).atLeastOnce();
-expect(shellState.getConnector()).andReturn(conn).atLeastOnce();
-expect(conn.tableOperations()).andReturn(tableOperations);
++expect(shellState.getAccumuloClient()).andReturn(client).atLeastOnce();
++expect(client.tableOperations()).andReturn(tableOperations);
shellState.checkTableState();
- expectLastCall().andVoid();
-
- // Table exists
- EasyMock.expect(client.tableOperations()).andReturn(tableOperations);
+ expectLastCall().once();
tableOperations.importDirectory("tablename", "in_dir", "fail_dir", false);
- expectLastCall().times(3);
-
- EasyMock.replay(client, cli, shellState, tableOperations);
+ expectLastCall().once();
-replay(conn, cli, shellState, tableOperations);
++replay(client, cli, shellState, tableOperations);
cmd.execute("importdirectory in_dir fail_dir false", cli, shellState);
-
}
/**
@@@ -94,37 -93,23 +93,23 @@@
*/
@Test
public void testPassTableOptCmdForm() throws Exception {
-
String[] cliArgs = {"in_dir", "fail_dir", "false"};
- //
- // EasyMock.expect(cli.hasOption('t')).andReturn(false);
- EasyMock.expect(cli.hasOption("t")).andReturn(true);
- EasyMock.expect(cli.hasOption("t")).andReturn(true);
- EasyMock.expect(cli.getOptionValue("t")).andReturn("passedName");
+ // -t option specified, table is from option
+ expect(cli.hasOption("t")).andReturn(true).once();
+ expect(cli.getOptionValue("t")).andReturn("passedName").once();
+ expect(tableOperations.exists("passedName")).andReturn(true).once();
- EasyMock.expect(tableOperations.exists("passedName")).andReturn(true);
- EasyMock.expect(shellState.getAccumuloClient()).andReturn(client);
- EasyMock.expect(client.tableOperations()).andReturn(tableOperations);
+ expect(cli.getArgs()).andReturn(cliArgs).atLeastOnce();
-
[accumulo] 02/02: Use newer importDirectory method for unit test of #1299
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6aa4c5f643d7e4fa26567403a09fc4b5e180b1a8
Author: Christopher Tubbs
AuthorDate: Thu Aug 15 22:33:11 2019 -0400
Use newer importDirectory method for unit test of #1299
Switch ImportDirectoryCommandTest to use the newer (non-deprecated) bulk
import API, and ensure the mock object is verified.
---
.../shell/commands/ImportDirectoryCommandTest.java | 27 ++
1 file changed, 18 insertions(+), 9 deletions(-)
diff --git
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
index 556ad5a..3d075d6 100644
---
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
@@ -24,6 +24,7 @@ import static org.easymock.EasyMock.verify;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.clientImpl.bulk.BulkImport;
import org.apache.accumulo.shell.Shell;
import org.apache.commons.cli.CommandLine;
import org.junit.After;
@@ -38,6 +39,7 @@ public class ImportDirectoryCommandTest {
private CommandLine cli;
private Shell shellState;
private TableOperations tableOperations;
+ private BulkImport bulkImport;
@Before
public void setup() {
@@ -50,11 +52,12 @@ public class ImportDirectoryCommandTest {
cli = createMock(CommandLine.class);
shellState = createMock(Shell.class);
tableOperations = createMock(TableOperations.class);
+bulkImport = createMock(BulkImport.class);
}
@After
public void verifyMocks() {
-verify(client, cli, shellState, tableOperations);
+verify(client, cli, shellState, tableOperations, bulkImport);
}
/**
@@ -65,7 +68,7 @@ public class ImportDirectoryCommandTest {
*/
@Test
public void testOriginalCmdForm() throws Exception {
-String[] cliArgs = {"in_dir", "fail_dir", "false"};
+String[] cliArgs = {"in_dir", "false"};
// no -t option, use current table context
expect(cli.hasOption("t")).andReturn(false).once();
@@ -78,11 +81,14 @@ public class ImportDirectoryCommandTest {
shellState.checkTableState();
expectLastCall().once();
-tableOperations.importDirectory("tablename", "in_dir", "fail_dir", false);
+
expect(tableOperations.importDirectory("in_dir")).andReturn(bulkImport).once();
+expect(bulkImport.to("tablename")).andReturn(bulkImport).once();
+expect(bulkImport.tableTime(false)).andReturn(bulkImport).once();
+bulkImport.load();
expectLastCall().once();
-replay(client, cli, shellState, tableOperations);
-cmd.execute("importdirectory in_dir fail_dir false", cli, shellState);
+replay(client, cli, shellState, tableOperations, bulkImport);
+cmd.execute("importdirectory in_dir false", cli, shellState);
}
/**
@@ -93,7 +99,7 @@ public class ImportDirectoryCommandTest {
*/
@Test
public void testPassTableOptCmdForm() throws Exception {
-String[] cliArgs = {"in_dir", "fail_dir", "false"};
+String[] cliArgs = {"in_dir", "false"};
// -t option specified, table is from option
expect(cli.hasOption("t")).andReturn(true).once();
@@ -106,10 +112,13 @@ public class ImportDirectoryCommandTest {
// shellState.checkTableState() is NOT called
-tableOperations.importDirectory("passedName", "in_dir", "fail_dir", false);
+
expect(tableOperations.importDirectory("in_dir")).andReturn(bulkImport).once();
+expect(bulkImport.to("passedName")).andReturn(bulkImport).once();
+expect(bulkImport.tableTime(false)).andReturn(bulkImport).once();
+bulkImport.load();
expectLastCall().once();
-replay(client, cli, shellState, tableOperations);
-cmd.execute("importdirectory in_dir fail_dir false", cli, shellState);
+replay(client, cli, shellState, tableOperations, bulkImport);
+cmd.execute("importdirectory in_dir false", cli, shellState);
}
}
[accumulo-testing] branch master updated: Drop example files (#100)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/master by this push:
new f491c91 Drop example files (#100)
f491c91 is described below
commit f491c91817c24ac36fa9ff316b4d07114b237a5f
Author: Mike Miller
AuthorDate: Thu Aug 15 17:51:29 2019 -0400
Drop example files (#100)
* Drop .example files
* Remove references to these files from documentation
---
Dockerfile | 1 -
README.md | 7 +++
bin/build | 10 +++---
bin/performance| 7 ++-
conf/.gitignore| 3 ---
...-testing.properties.example => accumulo-testing.properties} | 0
conf/{env.sh.example => env.sh}| 7 ++-
conf/{log4j.properties.example => log4j.properties}| 0
test/scalability/README.md | 8
test/scalability/conf/{Ingest.conf.example => Ingest.conf} | 0
test/scalability/conf/{site.conf.example => site.conf} | 0
test/stress/README.md | 7 +++
test/stress/{stress-env.sh.example => stress-env.sh} | 0
13 files changed, 17 insertions(+), 33 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 2437f66..8c86058 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -37,7 +37,6 @@ RUN mkdir /opt/at/conf
COPY ./conf/accumulo-client.properties /opt/at/conf/
COPY ./conf/accumulo-testing.properties /opt/at/conf/
COPY ./conf/log4j.properties* /opt/at/conf/
-RUN if [[ ! -f /opt/at/conf/log4j.properties ]]; then mv
/opt/at/conf/log4j.properties.example /opt/at/conf/log4j.properties; fi
RUN touch /opt/at/conf/env.sh
COPY ./bin/cingest /opt/at/bin
diff --git a/README.md b/README.md
index f393aa2..88b0009 100644
--- a/README.md
+++ b/README.md
@@ -15,12 +15,11 @@ on your machine as well as an Accumulo instance to use for
testing.
cd accumulo-testing
2. All configuation files for the test suite are in `conf/`. Only the
`accumulo-testing.properties`
- configuration file needs to be created and edited as all other
configuration files are optional.
+ configuration file needs to be edited as all other configuration files are
optional.
In `accumulo-testing.properites`, review the properties with
`test.common.*` prefix as these are
used by all tests.
cd conf/
-cp accumulo-testing.properties.example accumulo-testing.properties
vim accumulo-testing.properties
@@ -47,8 +46,8 @@ run in Docker:
1. To create the `accumulo-testing` docker image, make sure the following
files exist in your clone:
-* `conf/accumulo-client.properties` - Copy and configure this from your
Accumulo install
-* `conf/accumulo-testing.properties` - Copy this from the example file and
configure it
+* `conf/accumulo-client.properties` - Configure this file from your
Accumulo install
+* `conf/accumulo-testing.properties` - Configure this file for testing
* `target/accumulo-testing-2.0.0-SNAPSHOT-shaded.jar` - Can be created
using `./bin/build`
Run the following command to create the image. `HADOOP_HOME` should be
where Hadoop is installed on your cluster.
diff --git a/bin/build b/bin/build
index 457e87a..d5d178e 100755
--- a/bin/build
+++ b/bin/build
@@ -18,10 +18,6 @@
bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
at_home=$( cd "$( dirname "$bin_dir" )" && pwd )
-if [ -f "$at_home/conf/env.sh" ]; then
- conf="$at_home/conf/env.sh"
-else
- conf="$at_home/conf/env.sh.example"
-fi
-echo "Using $conf for setup"
-source $conf
+env_sh="$at_home/conf/env.sh"
+echo "Using $env_sh for setup"
+source $env_sh
diff --git a/bin/performance b/bin/performance
index b6ed57c..6b5f25d 100755
--- a/bin/performance
+++ b/bin/performance
@@ -45,11 +45,8 @@ function build_shade_jar() {
log4j_config="$at_home/conf/log4j.properties"
if [ ! -f "$log4j_config" ]; then
- log4j_config="$at_home/conf/log4j.properties.example"
- if [ ! -f "$log4j_config" ]; then
-echo "Could not find logj4.properties or log4j.properties.example in
$at_home/conf"
-exit 1
- fi
+ echo "Could not find logj4.properties in $at_home/conf"
+ exit 1
fi
diff --git a/conf/.gitignore b/conf/.gitignore
index bd1d6b3..b0aa095 100644
--- a/conf/.gitignore
+++ b/conf/.gitignore
@@ -1,5 +1,2 @@
/accumulo-client.properties
-/accumulo-testing.properties
-/env.sh
-/log4j.properties
/cluster-control.sh
diff --git a/conf/accumulo-testing.properties.example
b/conf/accumulo-testing.properties
similarity index 100%
rename from conf/accumulo-testing.properties.example
rename to
[accumulo] branch 2.0 updated (6ae7ddb -> a6b72c5)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 6ae7ddb Merge branch '1.9' into 2.0 add 2d0091d Add optional -t tablename to importdirectory shell command. (#1299) add d2803b5 Merge branch '1.9' into 2.0 add a6b72c5 Merge 1.9 into 2.0 No new revisions were added by this update. Summary of changes: .../shell/commands/ImportDirectoryCommand.java | 30 +++-- .../shell/commands/ImportDirectoryCommandTest.java | 130 + .../org/apache/accumulo/test/ShellServerIT.java| 61 ++ 3 files changed, 211 insertions(+), 10 deletions(-) create mode 100644 shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
[accumulo] branch master updated (c876275 -> 43a7629)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from c876275 Merge branch '2.0' add 2d0091d Add optional -t tablename to importdirectory shell command. (#1299) new d2803b5 Merge branch '1.9' into 2.0 new a6b72c5 Merge 1.9 into 2.0 new 43a7629 Merge branch '2.0' The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../shell/commands/ImportDirectoryCommand.java | 30 +++-- .../shell/commands/ImportDirectoryCommandTest.java | 130 + .../org/apache/accumulo/test/ShellServerIT.java| 61 ++ 3 files changed, 211 insertions(+), 10 deletions(-) create mode 100644 shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
[accumulo] 03/03: Merge branch '2.0'
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 43a762938acf724a8117025a69224bbd9dcdc061 Merge: c876275 a6b72c5 Author: Ed Coleman AuthorDate: Thu Aug 15 16:58:24 2019 -0400 Merge branch '2.0' .../shell/commands/ImportDirectoryCommand.java | 30 +++-- .../shell/commands/ImportDirectoryCommandTest.java | 130 + .../org/apache/accumulo/test/ShellServerIT.java| 61 ++ 3 files changed, 211 insertions(+), 10 deletions(-)
[accumulo] 01/03: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit d2803b5cdf024c78ddde9e899ff7470f0f44d03d
Merge: 6ae7ddb 2d0091d
Author: Ed Coleman
AuthorDate: Thu Aug 15 16:01:35 2019 -0400
Merge branch '1.9' into 2.0
.../shell/commands/ImportDirectoryCommand.java | 26 +++--
.../shell/commands/ImportDirectoryCommandTest.java | 130 +
.../org/apache/accumulo/test/ShellServerIT.java| 62 ++
3 files changed, 210 insertions(+), 8 deletions(-)
diff --cc
shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
index cbee631,dc92293..ac5a30f
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
@@@ -29,43 -30,24 +30,45 @@@ public class ImportDirectoryCommand ext
@Override
public String description() {
- return "bulk imports an entire directory of data files to the current"
- + " table. The boolean argument determines if accumulo sets the time.
"
- + "Passing 3 arguments will use the old bulk import. The new bulk
import only takes 2 "
- + "arguments: true|false";
+ return "bulk imports an entire directory of data files into an existing
table."
+ + " The table is either passed with the -t tablename opt, or into to
the current"
-+ " table if the -t option is not provided. The boolean argument
determines if accumulo sets the time.";
+++ " table if the -t option is not provided. The boolean argument
determines if accumulo"
+++ " sets the time. Passing 3 arguments will use the old bulk import.
The new bulk"
+++ " import only takes 2 arguments: true|false";
}
+ @SuppressWarnings("deprecation")
@Override
public int execute(final String fullCommand, final CommandLine cl, final
Shell shellState)
throws IOException, AccumuloException, AccumuloSecurityException,
TableNotFoundException {
- shellState.checkTableState();
+
+ final String tableName = OptUtil.getTableOpt(cl, shellState);
-String dir = cl.getArgs()[0];
-String failureDir = cl.getArgs()[1];
+String[] args = cl.getArgs();
+String dir = args[0];
+boolean setTime;
-final boolean setTime = Boolean.parseBoolean(cl.getArgs()[2]);
-
-shellState.getConnector().tableOperations().importDirectory(tableName,
dir, failureDir,
-setTime);
+// new bulk import only takes 2 args
+if (args.length == 2) {
+ setTime = Boolean.parseBoolean(cl.getArgs()[1]);
+ shellState.getAccumuloClient().tableOperations().importDirectory(dir)
- .to(shellState.getTableName()).tableTime(setTime).load();
++ .to(tableName).tableTime(setTime).load();
+} else if (args.length == 3) {
+ // warn using deprecated bulk import
+ Shell.log.warn(
+ "Deprecated since 2.0.0. New bulk import technique does not take a
failure directory "
+ + "as an argument.");
+ String failureDir = args[1];
+ setTime = Boolean.parseBoolean(cl.getArgs()[2]);
-
shellState.getAccumuloClient().tableOperations().importDirectory(shellState.getTableName(),
++
shellState.getAccumuloClient().tableOperations().importDirectory(tableName,
+ dir, failureDir, setTime);
+ return 0;
+} else {
+ shellState.printException(
+ new IllegalArgumentException(String.format("Expected 2 or 3
arguments. There %s %d.",
+ args.length == 1 ? "was" : "were", args.length)));
+ printHelp(shellState);
+}
return 0;
}
@@@ -77,7 -60,14 +80,14 @@@
@Override
public String usage() {
- return getName() + " [failureDirectory] true|false";
-return getName() + "[-t tablename]
true|false";
++return getName() + "[-t tablename] [failureDirectory]
true|false";
+ }
+
+ @Override
+ public Options getOptions() {
+ final Options opts = super.getOptions();
+ opts.addOption(OptUtil.tableOpt("name of the table to import files
into"));
+ return opts;
}
}
[accumulo] 02/03: Merge 1.9 into 2.0
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a6b72c5ae7f704fdeb683929667fc7c6b0e79b3b
Author: Ed Coleman
AuthorDate: Thu Aug 15 16:57:41 2019 -0400
Merge 1.9 into 2.0
---
.../shell/commands/ImportDirectoryCommand.java | 8
.../shell/commands/ImportDirectoryCommandTest.java | 22 +++---
.../org/apache/accumulo/test/ShellServerIT.java| 1 -
3 files changed, 15 insertions(+), 16 deletions(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
index ac5a30f..5decaed 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
@@ -51,8 +51,8 @@ public class ImportDirectoryCommand extends Command {
// new bulk import only takes 2 args
if (args.length == 2) {
setTime = Boolean.parseBoolean(cl.getArgs()[1]);
- shellState.getAccumuloClient().tableOperations().importDirectory(dir)
- .to(tableName).tableTime(setTime).load();
+
shellState.getAccumuloClient().tableOperations().importDirectory(dir).to(tableName)
+ .tableTime(setTime).load();
} else if (args.length == 3) {
// warn using deprecated bulk import
Shell.log.warn(
@@ -60,8 +60,8 @@ public class ImportDirectoryCommand extends Command {
+ "as an argument.");
String failureDir = args[1];
setTime = Boolean.parseBoolean(cl.getArgs()[2]);
-
shellState.getAccumuloClient().tableOperations().importDirectory(tableName,
- dir, failureDir, setTime);
+
shellState.getAccumuloClient().tableOperations().importDirectory(tableName,
dir, failureDir,
+ setTime);
return 0;
} else {
shellState.printException(
diff --git
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
index c819d6c..963d919 100644
---
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
@@ -18,7 +18,7 @@ package org.apache.accumulo.shell.commands;
import static org.easymock.EasyMock.expectLastCall;
-import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.shell.Shell;
import org.apache.commons.cli.CommandLine;
@@ -30,7 +30,7 @@ public class ImportDirectoryCommandTest {
private ImportDirectoryCommand cmd;
- private Connector conn;
+ private AccumuloClient client;
private CommandLine cli;
private Shell shellState;
private TableOperations tableOperations;
@@ -42,7 +42,7 @@ public class ImportDirectoryCommandTest {
// Initialize that internal state
cmd.getOptions();
-conn = EasyMock.createMock(Connector.class);
+client = EasyMock.createMock(AccumuloClient.class);
cli = EasyMock.createMock(CommandLine.class);
shellState = EasyMock.createMock(Shell.class);
tableOperations = EasyMock.createMock(TableOperations.class);
@@ -68,19 +68,19 @@ public class ImportDirectoryCommandTest {
EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
-EasyMock.expect(shellState.getConnector()).andReturn(conn);
+EasyMock.expect(shellState.getAccumuloClient()).andReturn(client);
EasyMock.expect(shellState.getTableName()).andReturn("tablename");
shellState.checkTableState();
expectLastCall().andVoid();
// Table exists
-EasyMock.expect(conn.tableOperations()).andReturn(tableOperations);
+EasyMock.expect(client.tableOperations()).andReturn(tableOperations);
tableOperations.importDirectory("tablename", "in_dir", "fail_dir", false);
expectLastCall().times(3);
-EasyMock.replay(conn, cli, shellState, tableOperations);
+EasyMock.replay(client, cli, shellState, tableOperations);
cmd.execute("importdirectory in_dir fail_dir false", cli, shellState);
@@ -104,25 +104,25 @@ public class ImportDirectoryCommandTest {
EasyMock.expect(cli.getOptionValue("t")).andReturn("passedName");
EasyMock.expect(tableOperations.exists("passedName")).andReturn(true);
-EasyMock.expect(shellState.getConnector()).andReturn(conn);
-EasyMock.expect(conn.tableOperations()).andReturn(tableOperations);
+EasyMock.expect(shellState.getAccumuloClient()).andReturn(client);
+EasyMock.expect(client.tableOperations()).andReturn(tableOperations);
EasyMock.expect(cli.getArgs()).andReturn(cliArgs);
[accumulo] branch 2.0 updated (95c92ed -> 6ae7ddb)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 95c92ed Merge branch '1.9' into 2.0 add ab90032 Fix #1314 fixes gc thread issue by creating one timed task instance (#1318) new 6ae7ddb Merge branch '1.9' into 2.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 10 -- .../org/apache/accumulo/gc/SimpleGarbageCollector.java | 15 ++- .../accumulo/gc/GarbageCollectWriteAheadLogsTest.java | 18 ++ 3 files changed, 36 insertions(+), 7 deletions(-)
[accumulo] branch master updated (f39c2fe -> c876275)
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from f39c2fe Merge branch '2.0' add ab90032 Fix #1314 fixes gc thread issue by creating one timed task instance (#1318) new 6ae7ddb Merge branch '1.9' into 2.0 new c876275 Merge branch '2.0' The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 10 -- .../org/apache/accumulo/gc/SimpleGarbageCollector.java | 15 ++- .../accumulo/gc/GarbageCollectWriteAheadLogsTest.java | 18 ++ 3 files changed, 36 insertions(+), 7 deletions(-)
[accumulo] 01/01: Merge branch '2.0'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit c876275d421d8e4bf3f22321294f0cf4a72caad1
Merge: f39c2fe 6ae7ddb
Author: Christopher Tubbs
AuthorDate: Thu Aug 15 15:45:42 2019 -0400
Merge branch '2.0'
.../accumulo/gc/GarbageCollectWriteAheadLogs.java | 10 --
.../org/apache/accumulo/gc/SimpleGarbageCollector.java | 15 ++-
.../accumulo/gc/GarbageCollectWriteAheadLogsTest.java | 18 ++
3 files changed, 36 insertions(+), 7 deletions(-)
diff --cc
server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 5637a8d,e671219..2858bf6
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -89,13 -90,9 +90,9 @@@ public class GarbageCollectWriteAheadLo
this.context = context;
this.fs = fs;
this.useTrash = useTrash;
- this.liveServers = new LiveTServerSet(context, (current, deleted, added)
-> {
- log.debug("New tablet servers noticed: {}", added);
- log.debug("Tablet servers removed: {}", deleted);
- });
- liveServers.startListeningForTabletServerChanges();
+ this.liveServers = liveServers;
this.walMarker = new WalStateManager(context);
-this.store = () -> Iterators.concat(new
ZooTabletStateStore(context).iterator(),
+this.store = () -> Iterators.concat(new
ZooTabletStateStore(context.getAmple()).iterator(),
new RootTabletStateStore(context).iterator(), new
MetaDataStateStore(context).iterator());
}
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6ae7ddb99d10635e713742f35fc1db0ce9332ba2
Merge: 95c92ed ab90032
Author: Christopher Tubbs
AuthorDate: Thu Aug 15 15:45:02 2019 -0400
Merge branch '1.9' into 2.0
.../accumulo/gc/GarbageCollectWriteAheadLogs.java | 10 --
.../org/apache/accumulo/gc/SimpleGarbageCollector.java | 15 ++-
.../accumulo/gc/GarbageCollectWriteAheadLogsTest.java | 18 ++
3 files changed, 36 insertions(+), 7 deletions(-)
diff --cc
server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 289e69d,7afcdc8..e671219
---
a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++
b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@@ -85,18 -91,26 +85,15 @@@ public class GarbageCollectWriteAheadLo
* @param useTrash
* true to move files to trash rather than delete them
*/
- GarbageCollectWriteAheadLogs(final ServerContext context, VolumeManager fs,
boolean useTrash) {
- GarbageCollectWriteAheadLogs(final AccumuloServerContext context, final
VolumeManager fs,
- final LiveTServerSet liveServers, boolean useTrash) throws IOException {
++ GarbageCollectWriteAheadLogs(final ServerContext context, final
VolumeManager fs,
++ final LiveTServerSet liveServers, boolean useTrash) {
this.context = context;
this.fs = fs;
this.useTrash = useTrash;
- this.liveServers = new LiveTServerSet(context, (current, deleted, added)
-> {
- log.debug("New tablet servers noticed: {}", added);
- log.debug("Tablet servers removed: {}", deleted);
- });
- liveServers.startListeningForTabletServerChanges();
+ this.liveServers = liveServers;
-
-this.walMarker = new WalStateManager(context.getInstance(),
ZooReaderWriter.getInstance());
-this.store = new Iterable() {
- @Override
- public Iterator iterator() {
-try {
- return Iterators.concat(new ZooTabletStateStore().iterator(),
- new RootTabletStateStore(context).iterator(),
- new MetaDataStateStore(context).iterator());
-} catch (DistributedStoreException e) {
- throw new RuntimeException(e);
-}
- }
-};
+this.walMarker = new WalStateManager(context);
+this.store = () -> Iterators.concat(new
ZooTabletStateStore(context).iterator(),
+new RootTabletStateStore(context).iterator(), new
MetaDataStateStore(context).iterator());
}
/**
@@@ -124,34 -138,31 +121,35 @@@
}
public void collect(GCStatus status) {
-
-Span span = Trace.start("getCandidates");
try {
- status.currentLog.started = System.currentTimeMillis();
-
- Map recoveryLogs = getSortedWALogs();
-
- Map> logsByServer = new HashMap<>();
- Map> logsState = new HashMap<>();
- // Scan for log file info first: the order is important
- // Consider:
- // * get live servers
- // * new server gets a lock, creates a log
- // * get logs
- // * the log appears to belong to a dead server
- long count = getCurrent(logsByServer, logsState);
- long fileScanStop = System.currentTimeMillis();
-
- log.info(String.format("Fetched %d files for %d servers in %.2f
seconds", count,
- logsByServer.size(), (fileScanStop - status.currentLog.started) /
1000.));
- status.currentLog.candidates = count;
- span.stop();
+ long count;
+ long fileScanStop;
+ Map> logsByServer;
+ Map> logsState;
+ Map recoveryLogs;
+ try (TraceScope span = Trace.startSpan("getCandidates")) {
+status.currentLog.started = System.currentTimeMillis();
+
+recoveryLogs = getSortedWALogs();
+
+logsByServer = new HashMap<>();
+logsState = new HashMap<>();
+// Scan for log file info first: the order is important
+// Consider:
+// * get live servers
+// * new server gets a lock, creates a log
+// * get logs
+// * the log appears to belong to a dead server
+count = getCurrent(logsByServer, logsState);
+fileScanStop = System.currentTimeMillis();
+
+log.info(String.format("Fetched %d files for %d servers in %.2f
seconds", count,
+logsByServer.size(), (fileScanStop - status.currentLog.started) /
1000.));
+status.currentLog.candidates = count;
+ }
// now it's safe to get the liveServers
+ liveServers.scanServers();
Set currentServers = liveServers.getCurrentServers();
Map uuidToTServer;
diff --cc
server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 147239a,24ce61c..2a0a19f
---
[accumulo] branch 1.9 updated: Add optional -t tablename to importdirectory shell command. (#1299)
This is an automated email from the ASF dual-hosted git repository.
edcoleman pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push:
new 2d0091d Add optional -t tablename to importdirectory shell command.
(#1299)
2d0091d is described below
commit 2d0091d7136c216e1e043d462c34870f4e68ba55
Author: EdColeman
AuthorDate: Thu Aug 15 15:41:15 2019 -0400
Add optional -t tablename to importdirectory shell command. (#1299)
* Add optional -t tablename to importdirectory shell command.
the -t tablename is optional to preserve original behaviour. Without
the option, the current table is used.
* added getOptions
* wip - tests cmd format, need to evaluate testing functionality
---
.../shell/commands/ImportDirectoryCommand.java | 25 +++-
.../shell/commands/ImportDirectoryCommandTest.java | 130 +
.../org/apache/accumulo/test/ShellServerIT.java| 62 ++
3 files changed, 211 insertions(+), 6 deletions(-)
diff --git
a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
index ec1576b..dc92293 100644
---
a/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
+++
b/shell/src/main/java/org/apache/accumulo/shell/commands/ImportDirectoryCommand.java
@@ -24,37 +24,50 @@ import
org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.shell.Shell;
import org.apache.accumulo.shell.Shell.Command;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
public class ImportDirectoryCommand extends Command {
@Override
public String description() {
-return "bulk imports an entire directory of data files to the current"
-+ " table. The boolean argument determines if accumulo sets the time.";
+return "bulk imports an entire directory of data files into an existing
table."
++ " The table is either passed with the -t tablename opt, or into to
the current"
++ " table if the -t option is not provided. The boolean argument
determines if accumulo sets the time.";
}
@Override
public int execute(final String fullCommand, final CommandLine cl, final
Shell shellState)
throws IOException, AccumuloException, AccumuloSecurityException,
TableNotFoundException {
-shellState.checkTableState();
+
+final String tableName = OptUtil.getTableOpt(cl, shellState);
String dir = cl.getArgs()[0];
String failureDir = cl.getArgs()[1];
+
final boolean setTime = Boolean.parseBoolean(cl.getArgs()[2]);
-
shellState.getConnector().tableOperations().importDirectory(shellState.getTableName(),
dir,
-failureDir, setTime);
+shellState.getConnector().tableOperations().importDirectory(tableName,
dir, failureDir,
+setTime);
+
return 0;
}
@Override
public int numArgs() {
+// arg count for args not handled with Options
return 3;
}
@Override
public String usage() {
-return getName() + " true|false";
+return getName() + "[-t tablename]
true|false";
+ }
+
+ @Override
+ public Options getOptions() {
+final Options opts = super.getOptions();
+opts.addOption(OptUtil.tableOpt("name of the table to import files into"));
+return opts;
}
}
diff --git
a/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
new file mode 100644
index 000..c819d6c
--- /dev/null
+++
b/shell/src/test/java/org/apache/accumulo/shell/commands/ImportDirectoryCommandTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.shell.commands;
+
+import static org.easymock.EasyMock.expectLastCall;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.shell.Shell;
+import org.apache.commons.cli.CommandLine;
+import
[accumulo] branch 1.9 updated (556c29e -> ab90032)
This is an automated email from the ASF dual-hosted git repository. edcoleman pushed a change to branch 1.9 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 556c29e fix #1326 Copy instead of clone metadata table in IT add ab90032 Fix #1314 fixes gc thread issue by creating one timed task instance (#1318) No new revisions were added by this update. Summary of changes: .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 17 + .../apache/accumulo/gc/SimpleGarbageCollector.java | 21 - .../gc/GarbageCollectWriteAheadLogsTest.java| 18 ++ 3 files changed, 43 insertions(+), 13 deletions(-)
[accumulo] 01/01: Merge branch '2.0'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f39c2fef1c1c911e86985ce6352c5d1bc9967c5b Merge: bea0b7a 95c92ed Author: Keith Turner AuthorDate: Thu Aug 15 14:05:00 2019 -0400 Merge branch '2.0' .../functional/TabletStateChangeIteratorIT.java| 73 -- 1 file changed, 53 insertions(+), 20 deletions(-)
[accumulo] branch master updated (bea0b7a -> f39c2fe)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from bea0b7a Merge branch '2.0' add 556c29e fix #1326 Copy instead of clone metadata table in IT add 95c92ed Merge branch '1.9' into 2.0 new f39c2fe Merge branch '2.0' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../functional/TabletStateChangeIteratorIT.java| 73 -- 1 file changed, 53 insertions(+), 20 deletions(-)
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.0
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 95c92ed95404ec1663ea97d8b4b00d58c26ba762
Merge: f63da3d 556c29e
Author: Keith Turner
AuthorDate: Thu Aug 15 14:01:51 2019 -0400
Merge branch '1.9' into 2.0
.../functional/TabletStateChangeIteratorIT.java| 73 --
1 file changed, 53 insertions(+), 20 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index b83e735,2fd8589..35c53cc
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@@ -35,7 -34,10 +36,9 @@@ import org.apache.accumulo.core.client.
import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
+ import org.apache.accumulo.core.client.IsolatedScanner;
import org.apache.accumulo.core.client.MutationsRejectedException;
+ import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
@@@ -81,70 -81,70 +84,75 @@@ public class TabletStateChangeIteratorI
@Test
public void test() throws AccumuloException, AccumuloSecurityException,
TableExistsException,
TableNotFoundException {
-String[] tables = getUniqueNames(6);
-final String t1 = tables[0];
-final String t2 = tables[1];
-final String t3 = tables[2];
-final String metaCopy1 = tables[3];
-final String metaCopy2 = tables[4];
-final String metaCopy3 = tables[5];
-
-// create some metadata
-createTable(t1, true);
-createTable(t2, false);
-createTable(t3, true);
-
-// examine a clone of the metadata table, so we can manipulate it
-copyTable(MetadataTable.NAME, metaCopy1);
-
-State state = new State();
-while (findTabletsNeedingAttention(metaCopy1, state) > 0) {
- UtilWaitThread.sleep(500);
- copyTable(MetadataTable.NAME, metaCopy1);
-}
-assertEquals("No tables should need attention", 0,
-findTabletsNeedingAttention(metaCopy1, state));
-
-// The metadata table stabilized and metaCopy1 contains a copy suitable
for testing. Before
-// metaCopy1 is modified, copy it for subsequent test.
-copyTable(metaCopy1, metaCopy2);
-copyTable(metaCopy1, metaCopy3);
-
-// test the assigned case (no location)
-removeLocation(metaCopy1, t3);
-assertEquals("Should have two tablets without a loc", 2,
-findTabletsNeedingAttention(metaCopy1, state));
-
-// test the cases where the assignment is to a dead tserver
-reassignLocation(metaCopy2, t3);
-assertEquals("Should have one tablet that needs to be unassigned", 1,
-findTabletsNeedingAttention(metaCopy2, state));
-
-// test the cases where there is ongoing merges
-state = new State() {
- @Override
- public Collection merges() {
-String tableIdToModify =
getConnector().tableOperations().tableIdMap().get(t3);
-return Collections.singletonList(
-new MergeInfo(new KeyExtent(tableIdToModify, null, null),
MergeInfo.Operation.MERGE));
+
+try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+
- String[] tables = getUniqueNames(4);
++ String[] tables = getUniqueNames(6);
+ final String t1 = tables[0];
+ final String t2 = tables[1];
+ final String t3 = tables[2];
- final String cloned = tables[3];
++ final String metaCopy1 = tables[3];
++ final String metaCopy2 = tables[4];
++ final String metaCopy3 = tables[5];
+
+ // create some metadata
+ createTable(client, t1, true);
+ createTable(client, t2, false);
+ createTable(client, t3, true);
+
+ // examine a clone of the metadata table, so we can manipulate it
- cloneMetadataTable(client, cloned);
++ copyTable(client, MetadataTable.NAME, metaCopy1);
+
+ State state = new State(client);
- while (findTabletsNeedingAttention(client, cloned, state) > 0) {
++ while (findTabletsNeedingAttention(client, metaCopy1, state) > 0) {
+UtilWaitThread.sleep(500);
++copyTable(client, MetadataTable.NAME, metaCopy1);
}
-};
-assertEquals("Should have 2 tablets that need to be chopped or
unassigned", 1,
-findTabletsNeedingAttention(metaCopy2, state));
-
-// test the bad tablet location state case (inconsistent metadata)
-state = new State();
-addDuplicateLocation(metaCopy3, t3);
-assertEquals("Should have 1 tablet that
[accumulo] branch 2.0 updated (f63da3d -> 95c92ed)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from f63da3d Merge branch '1.9' into 2.0 add 556c29e fix #1326 Copy instead of clone metadata table in IT new 95c92ed Merge branch '1.9' into 2.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../functional/TabletStateChangeIteratorIT.java| 73 -- 1 file changed, 53 insertions(+), 20 deletions(-)
[accumulo] branch 1.9 updated: fix #1326 Copy instead of clone metadata table in IT
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push:
new 556c29e fix #1326 Copy instead of clone metadata table in IT
556c29e is described below
commit 556c29ee60c80b1fa8464fa85f24be44d8e0608d
Author: Keith Turner
AuthorDate: Thu Aug 15 13:25:12 2019 -0400
fix #1326 Copy instead of clone metadata table in IT
After the changes in #1309 cloning of metdata table is no longer
allowed. TabletStateChangeIteratorIT relied on cloning and
was changed to copy instead.
Also the test was very sensitive to concurrent chnages in the metadata
table. Suspect that cloning used to introduce a delay that hid this.
The change from cloning to copying caused the test to fail often
because of these timing issues. To avoid this, the test was refactored
to tolerate concurrent changes to the metadata table.
---
.../functional/TabletStateChangeIteratorIT.java| 71 --
1 file changed, 52 insertions(+), 19 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 6a30292..2fd8589 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
@@ -34,7 +35,9 @@ import org.apache.accumulo.core.client.BatchDeleter;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IsolatedScanner;
import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
@@ -78,11 +81,13 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
@Test
public void test() throws AccumuloException, AccumuloSecurityException,
TableExistsException,
TableNotFoundException {
-String[] tables = getUniqueNames(4);
+String[] tables = getUniqueNames(6);
final String t1 = tables[0];
final String t2 = tables[1];
final String t3 = tables[2];
-final String cloned = tables[3];
+final String metaCopy1 = tables[3];
+final String metaCopy2 = tables[4];
+final String metaCopy3 = tables[5];
// create some metadata
createTable(t1, true);
@@ -90,25 +95,30 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
createTable(t3, true);
// examine a clone of the metadata table, so we can manipulate it
-cloneMetadataTable(cloned);
+copyTable(MetadataTable.NAME, metaCopy1);
State state = new State();
-while (findTabletsNeedingAttention(cloned, state) > 0) {
+while (findTabletsNeedingAttention(metaCopy1, state) > 0) {
UtilWaitThread.sleep(500);
+ copyTable(MetadataTable.NAME, metaCopy1);
}
-assertEquals("No tables should need attention", 0,
findTabletsNeedingAttention(cloned, state));
+assertEquals("No tables should need attention", 0,
+findTabletsNeedingAttention(metaCopy1, state));
+
+// The metadata table stabilized and metaCopy1 contains a copy suitable
for testing. Before
+// metaCopy1 is modified, copy it for subsequent test.
+copyTable(metaCopy1, metaCopy2);
+copyTable(metaCopy1, metaCopy3);
// test the assigned case (no location)
-removeLocation(cloned, t3);
+removeLocation(metaCopy1, t3);
assertEquals("Should have two tablets without a loc", 2,
-findTabletsNeedingAttention(cloned, state));
+findTabletsNeedingAttention(metaCopy1, state));
// test the cases where the assignment is to a dead tserver
-getConnector().tableOperations().delete(cloned);
-cloneMetadataTable(cloned);
-reassignLocation(cloned, t3);
+reassignLocation(metaCopy2, t3);
assertEquals("Should have one tablet that needs to be unassigned", 1,
-findTabletsNeedingAttention(cloned, state));
+findTabletsNeedingAttention(metaCopy2, state));
// test the cases where there is ongoing merges
state = new State() {
@@ -120,17 +130,16 @@ public class TabletStateChangeIteratorIT extends
AccumuloClusterHarness {
}
};
assertEquals("Should have 2 tablets that
[accumulo] 01/01: Merge branch '2.0'
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bea0b7a8a1bfde5f90bf680ecf31d2d32fb25c84 Merge: 503e030 f63da3d Author: Mike Miller AuthorDate: Thu Aug 15 10:09:30 2019 -0400 Merge branch '2.0'
[accumulo] branch master updated (503e030 -> bea0b7a)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 503e030 Merge branch '2.0' add e770862 Backport BulkIngest test. Closes #1288 (#1312) add f63da3d Merge branch '1.9' into 2.0 new bea0b7a Merge branch '2.0' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[accumulo] 01/01: Merge branch '1.9' into 2.0
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a commit to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f63da3d4ce709402feb2169eeda4c9c290ad5b7b Merge: c573676 e770862 Author: Mike Miller AuthorDate: Thu Aug 15 10:08:31 2019 -0400 Merge branch '1.9' into 2.0
[accumulo] branch 2.0 updated (c573676 -> f63da3d)
This is an automated email from the ASF dual-hosted git repository. mmiller pushed a change to branch 2.0 in repository https://gitbox.apache.org/repos/asf/accumulo.git. from c573676 Merge branch '1.9' into 2.0 add e770862 Backport BulkIngest test. Closes #1288 (#1312) new f63da3d Merge branch '1.9' into 2.0 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[accumulo] branch 1.9 updated: Backport BulkIngest test. Closes #1288 (#1312)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push:
new e770862 Backport BulkIngest test. Closes #1288 (#1312)
e770862 is described below
commit e770862036128d963e49e6c9715e703f9b69c270
Author: Mike Miller
AuthorDate: Thu Aug 15 09:52:57 2019 -0400
Backport BulkIngest test. Closes #1288 (#1312)
* Created classes for running the bulk generate job
* Created run-bulk-generate.sh
* Also updated the example test files
---
.../accumulo/test/continuous/BulkIngest.java | 132 ++
.../accumulo/test/continuous/ContinuousIngest.java | 11 +-
.../test/continuous/ContinuousInputFormat.java | 194 +
test/system/continuous/continuous-env.sh.example | 8 +
test/system/continuous/ingesters.txt.example | 4 +-
test/system/continuous/run-bulk-generate.sh| 48 +
test/system/continuous/walkers.txt.example | 4 +-
7 files changed, 393 insertions(+), 8 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/continuous/BulkIngest.java
b/test/src/main/java/org/apache/accumulo/test/continuous/BulkIngest.java
new file mode 100644
index 000..22638b3
--- /dev/null
+++ b/test/src/main/java/org/apache/accumulo/test/continuous/BulkIngest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.test.continuous;
+
+import java.io.BufferedOutputStream;
+import java.io.PrintStream;
+import java.util.Collection;
+import java.util.UUID;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.MapReduceClientOnDefaultTable;
+import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat;
+import
org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.util.Base64;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.validators.PositiveInteger;
+
+/**
+ * Bulk import a million random key value pairs. Same format as
ContinuousIngest and can be verified
+ * by running ContinuousVerify.
+ */
+public class BulkIngest extends Configured implements Tool {
+ static class Opts extends ContinuousOpts {
+@Parameter(names = "--dir", description = "the bulk dir to use", required
= true)
+String dir;
+
+@Parameter(names = "--reducers", description = "the number of reducers to
use",
+validateWith = PositiveInteger.class)
+int reducers = 10;
+
+@Parameter(names = "--mapTasks", description = "the number of map tasks to
use",
+validateWith = PositiveInteger.class)
+int mapTasks = 10;
+
+@Parameter(names = "--mapNodes",
+description = "the number of linked list key value nodes per mapper",
+validateWith = PositiveInteger.class)
+int mapNodes = 1000;
+ }
+
+ public static final Logger log = LoggerFactory.getLogger(BulkIngest.class);
+
+ @Override
+ public int run(String[] args) throws Exception {
+String ingestInstanceId = UUID.randomUUID().toString();
+
+Job job = Job.getInstance(getConf());
+job.setJobName("BulkIngest_" + ingestInstanceId);
+job.setJarByClass(BulkIngest.class);
+// very important to prevent guava conflicts
+job.getConfiguration().set("mapreduce.job.classloader", "true");
+FileSystem fs = FileSystem.get(job.getConfiguration());
+
+log.info(String.format("UUID %d %s", System.currentTimeMillis(),
ingestInstanceId));
+
+job.setInputFormatClass(ContinuousInputFormat.class);
+
+// map the generated random longs to key values
+job.setMapOutputKeyClass(Key.class);
+
