[accumulo] 01/01: Merge branch '1576-import-table-api'
This is an automated email from the ASF dual-hosted git repository. drew pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git commit e4596fee8d0800f1ad26b1b15cd922774f656c7d Merge: 7684a86 aedf445 Author: Drew Farris AuthorDate: Wed Apr 8 17:07:22 2020 -0400 Merge branch '1576-import-table-api' .../core/client/admin/TableOperations.java | 19 -- .../core/clientImpl/TableOperationsImpl.java | 24 +- .../apache/accumulo/core/util/ByteBufferUtil.java | 12 + .../core/clientImpl/TableOperationsHelperTest.java | 2 +- .../server/security/AuditedSecurityOperation.java | 9 --- .../server/security/SecurityOperation.java | 2 +- .../apache/accumulo/master/FateServiceHandler.java | 15 --- .../master/tableOps/tableImport/ImportTable.java | 29 +- .../tableOps/tableImport/ImportedTableInfo.java| 6 - .../tableOps/tableImport/ImportTableTest.java | 23 ++--- .../shell/commands/ImportTableCommand.java | 7 +- .../org/apache/accumulo/test/AuditMessageIT.java | 3 ++- .../org/apache/accumulo/test/ImportExportIT.java | 8 +++--- .../org/apache/accumulo/test/NamespacesIT.java | 5 ++-- 14 files changed, 106 insertions(+), 58 deletions(-)
[accumulo] branch master updated (7684a86 -> e4596fe)
This is an automated email from the ASF dual-hosted git repository. drew pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 7684a86 Remove errorprone profile from pom.xml (#1580) add aedf445 Closes #1576: Add importTable with directory list as Set (#1579) new e4596fe Merge branch '1576-import-table-api' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/client/admin/TableOperations.java | 19 -- .../core/clientImpl/TableOperationsImpl.java | 24 +- .../apache/accumulo/core/util/ByteBufferUtil.java | 12 + .../core/clientImpl/TableOperationsHelperTest.java | 2 +- .../server/security/AuditedSecurityOperation.java | 9 --- .../server/security/SecurityOperation.java | 2 +- .../apache/accumulo/master/FateServiceHandler.java | 15 --- .../master/tableOps/tableImport/ImportTable.java | 29 +- .../tableOps/tableImport/ImportedTableInfo.java| 6 - .../tableOps/tableImport/ImportTableTest.java | 23 ++--- .../shell/commands/ImportTableCommand.java | 7 +- .../org/apache/accumulo/test/AuditMessageIT.java | 3 ++- .../org/apache/accumulo/test/ImportExportIT.java | 8 +++--- .../org/apache/accumulo/test/NamespacesIT.java | 5 ++-- 14 files changed, 106 insertions(+), 58 deletions(-)
[accumulo] branch master updated (e49682e -> 27acec1)
This is an automated email from the ASF dual-hosted git repository. drew pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from e49682e Minor improvements / fix warning add 27acec1 Closes #1053 add multi-volume support to import table No new revisions were added by this update. Summary of changes: .../core/clientImpl/TableOperationsImpl.java | 57 - .../tableOps/tableImport/CreateImportDir.java | 47 +--- .../tableOps/tableImport/FinishImportTable.java| 8 +- .../tableImport/ImportPopulateZookeeper.java | 3 +- .../master/tableOps/tableImport/ImportTable.java | 57 +++-- .../tableOps/tableImport/ImportedTableInfo.java| 12 +- .../tableOps/tableImport/MapImportFileNames.java | 93 +++--- .../tableOps/tableImport/MoveExportedFiles.java| 95 --- .../tableImport/PopulateMetadataTable.java | 42 --- .../tableOps/tableImport/ImportTableTest.java | 134 + .../org/apache/accumulo/test/ImportExportIT.java | 26 ++-- 11 files changed, 421 insertions(+), 153 deletions(-) create mode 100644 server/master/src/test/java/org/apache/accumulo/master/tableOps/tableImport/ImportTableTest.java
[accumulo] branch master updated: Add PrintInfo options to dump full keys, apply formatter (#1537)
This is an automated email from the ASF dual-hosted git repository.
drew pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new d4542ef Add PrintInfo options to dump full keys, apply formatter
(#1537)
d4542ef is described below
commit d4542efa675554306c5e3842c4ad519d359dbb0f
Author: Drew Farris
AuthorDate: Fri Mar 13 15:51:35 2020 -0400
Add PrintInfo options to dump full keys, apply formatter (#1537)
* Add PrintInfo options to dump full keys, apply `BiFunction` to format output
---
.../apache/accumulo/core/file/rfile/PrintInfo.java | 48 +++---
1 file changed, 43 insertions(+), 5 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
index 7dbb151..65d1f3b 100644
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/PrintInfo.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.function.BiFunction;
import org.apache.accumulo.core.cli.ConfigOpts;
import org.apache.accumulo.core.crypto.CryptoServiceFactory;
@@ -40,6 +41,7 @@ import org.apache.accumulo.core.file.rfile.bcfile.Utils;
import org.apache.accumulo.core.summary.SummaryReader;
import org.apache.accumulo.core.util.LocalityGroupUtil;
import org.apache.accumulo.start.spi.KeywordExecutable;
+import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -61,6 +63,12 @@ public class PrintInfo implements KeywordExecutable {
static class Opts extends ConfigOpts {
@Parameter(names = {"-d", "--dump"}, description = "dump the key/value
pairs")
boolean dump = false;
+@Parameter(names = {"--fullKeys"},
+description = "dump full keys regardless of length, do no truncate,
implies --dump")
+boolean fullKeys = false;
+@Parameter(names = {"--formatter"},
+description = "specify a BiFunction class to apply
to rfile contents, implies --dump")
+String formatterClazz = null;
@Parameter(names = {"-v", "--vis"}, description = "show visibility
metrics")
boolean vis = false;
@Parameter(names = {"--visHash"}, description = "show visibilities as
hashes, implies -v")
@@ -138,6 +146,15 @@ public class PrintInfo implements KeywordExecutable {
return "Prints rfile info";
}
+ protected Class> getFormatter(String
formatterClazz)
+ throws ClassNotFoundException {
+@SuppressWarnings("unchecked")
+Class> clazz =
+(Class>)
this.getClass().getClassLoader()
+.loadClass(formatterClazz).asSubclass(BiFunction.class);
+return clazz;
+ }
+
@SuppressFBWarnings(value = "DM_EXIT",
justification = "System.exit is fine here because it's a utility class
executed by a main()")
@Override
@@ -148,6 +165,13 @@ public class PrintInfo implements KeywordExecutable {
System.err.println("No files were given");
System.exit(1);
}
+
+if ((opts.fullKeys || opts.dump) && opts.formatterClazz != null) {
+ System.err.println(
+ "--formatter argument is incompatible with --dump or --fullKeys,
specify either, not both.");
+ System.exit(1);
+}
+
var siteConfig = opts.getSiteConfiguration();
Configuration conf = new Configuration();
@@ -197,7 +221,8 @@ public class PrintInfo implements KeywordExecutable {
Map> localityGroupCF = null;
- if (opts.histogram || opts.dump || opts.vis || opts.hash ||
opts.keyStats) {
+ if (opts.histogram || opts.dump || opts.vis || opts.hash ||
opts.keyStats || opts.fullKeys
+ || !StringUtils.isEmpty(opts.formatterClazz)) {
localityGroupCF = iter.getLocalityGroupCF();
FileSKVIterator dataIter;
@@ -220,23 +245,36 @@ public class PrintInfo implements KeywordExecutable {
}
}
+BiFunction formatter = null;
+if (opts.formatterClazz != null) {
+ final Class> formatterClass =
+ getFormatter(opts.formatterClazz);
+ formatter = formatterClass.newInstance();
+} else if (opts.fullKeys) {
+ formatter = (key, value) -> key.toStringNoTruncate() + " -> " +
value;
+} else if (opts.dump) {
+ formatter = (key, value) -> key + " -> " + value;
+}
+
for (String lgName : localityGroupCF.keySet()) {
L
[accumulo] branch master updated (6f4c772 -> 25644cb)
This is an automated email from the ASF dual-hosted git repository. drew pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git. from 6f4c772 Improve use of DistributedCacheHelper. Closes #896 (#1035) new e92ef4e Fixes #1041 Provides proper volumes to chooser from import table command new 25644cb Merge branch '1.9' The 9889 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../master/tableOps/tableImport/PopulateMetadataTable.java | 12 ++-- .../master/tableOps/tableImport/ImportTableTest.java | 12 ++-- 2 files changed, 12 insertions(+), 12 deletions(-)
[accumulo] branch 1.9 updated: Fixes #1041 Provides proper volumes to chooser from import table command
This is an automated email from the ASF dual-hosted git repository.
drew pushed a commit to branch 1.9
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.9 by this push:
new e92ef4e Fixes #1041 Provides proper volumes to chooser from import
table command
e92ef4e is described below
commit e92ef4ecf17e8b24d145a746c019848e3ec85d9f
Author: Drew Farris
AuthorDate: Wed Mar 20 07:53:42 2019 -0700
Fixes #1041 Provides proper volumes to chooser from import table command
---
.../accumulo/master/tableOps/PopulateMetadataTable.java | 12 ++--
.../org/apache/accumulo/master/tableOps/ImportTableTest.java | 12 ++--
2 files changed, 12 insertions(+), 12 deletions(-)
diff --git
a/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
index 160480b..75c080a 100644
---
a/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
+++
b/server/master/src/main/java/org/apache/accumulo/master/tableOps/PopulateMetadataTable.java
@@ -109,7 +109,7 @@ class PopulateMetadataTable extends MasterRepo {
// hdfs://localhost:8020/path/to/accumulo/tables/...
final String bulkDir = tableInfo.importDir;
- final String[] tableDirs = ServerConstants.getTablesDirs();
+ final String[] volumes = ServerConstants.getBaseUris();
ZipEntry zipEntry;
while ((zipEntry = zis.getNextEntry()) != null) {
@@ -155,7 +155,7 @@ class PopulateMetadataTable extends MasterRepo {
UTF_8);
// Build up a full
hdfs://localhost:8020/accumulo/tables/$id/c-XXX
- String absolutePath = getClonedTabletDir(master, tableDirs,
tabletDir);
+ String absolutePath = getClonedTabletDir(master, volumes,
tabletDir);
m = new Mutation(metadataRow);
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
@@ -173,7 +173,7 @@ class PopulateMetadataTable extends MasterRepo {
UTF_8);
// Build up a full
hdfs://localhost:8020/accumulo/tables/$id/c-XXX
- String absolutePath = getClonedTabletDir(master, tableDirs,
tabletDir);
+ String absolutePath = getClonedTabletDir(master, volumes,
tabletDir);
m = new Mutation(metadataRow);
TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m,
@@ -220,12 +220,12 @@ class PopulateMetadataTable extends MasterRepo {
*
* @return An absolute, unique path for the imported table
*/
- protected String getClonedTabletDir(Master master, String[] tableDirs,
String tabletDir) {
+ protected String getClonedTabletDir(Master master, String[] volumes, String
tabletDir) {
// We can try to spread out the tablet dirs across all volumes
-String tableDir =
master.getFileSystem().choose(Optional.of(tableInfo.tableId), tableDirs);
+String volume =
master.getFileSystem().choose(Optional.of(tableInfo.tableId), volumes);
// Build up a full hdfs://localhost:8020/accumulo/tables/$id/c-XXX
-return tableDir + "/" + tableInfo.tableId + "/" + tabletDir;
+return volume + "/" + ServerConstants.TABLE_DIR + "/" + tableInfo.tableId
+ "/" + tabletDir;
}
@Override
diff --git
a/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
b/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
index 5e0a32b..e22d710 100644
---
a/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
+++
b/server/master/src/test/java/org/apache/accumulo/master/tableOps/ImportTableTest.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.master.tableOps;
import static org.junit.Assert.assertEquals;
import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.ServerConstants;
import org.apache.accumulo.server.fs.VolumeManager;
import org.easymock.EasyMock;
import org.junit.Test;
@@ -38,21 +39,20 @@ public class ImportTableTest {
iti.tableId = "5";
// Different volumes with different paths
-String[] tableDirs = new String[] {"hdfs://nn1:8020/apps/accumulo1/tables",
-"hdfs://nn2:8020/applications/accumulo/tables"};
+String[] volumes = new String[] {"hdfs://nn1:8020/apps/accumulo1",
+"hdfs://nn2:8020/applications/accumulo"};
// This needs to be unique WRT the importtable command
String tabletDir = "/c-0001";
EasyMock.expect(master.getFileSystem()).andReturn(volumeManager);
// Choose the 2nd element
-EasyMock.expect(volumeManager.choose(Optional.of(iti.tableId), tableDirs))
-.andReturn(tableDirs[1]);
+EasyMock.expect(volumeManager.c
accumulo git commit: ACCUMULO-4483 Added another test case
Repository: accumulo
Updated Branches:
refs/heads/1.8 69ca25fa0 -> 83fe980f4
ACCUMULO-4483 Added another test case
This forces the exception by reseeking the in memory map
iterator to the same location after forcing a switch.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/83fe980f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/83fe980f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/83fe980f
Branch: refs/heads/1.8
Commit: 83fe980f43864a951f5aa5222ab7428a0d4853ae
Parents: 69ca25f
Author: Ivan Bella <i...@bella.name>
Authored: Tue Oct 11 20:20:10 2016 -0400
Committer: Drew Farris <d...@apache.org>
Committed: Tue Oct 11 20:40:33 2016 -0400
--
.../accumulo/tserver/InMemoryMapTest.java | 26
1 file changed, 26 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/83fe980f/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
--
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
index 59dc68e..f813d42 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
@@ -141,6 +141,13 @@ public class InMemoryMapTest {
}
+ static void aeNoNext(SortedKeyValueIterator<Key,Value> dc, String row,
String column, int ts, String val) throws IOException {
+assertTrue(dc.hasTop());
+assertEquals(nk(row, column, ts), dc.getTopKey());
+assertEquals(new Value(val.getBytes()), dc.getTopValue());
+
+ }
+
static Set newCFSet(String... cfs) {
HashSet cfSet = new HashSet<>();
for (String cf : cfs) {
@@ -248,6 +255,25 @@ public class InMemoryMapTest {
}
@Test
+ public void testDecodeValueModification() throws Exception {
+// This test case is the fix for ACCUMULO-4483
+InMemoryMap imm = newInMemoryMap(false,
tempFolder.newFolder().getAbsolutePath());
+
+mutate(imm, "r1", "foo:cq1", 3, "");
+MemoryIterator ski1 = imm.skvIterator(null);
+
+imm.delete(0);
+
+ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET,
false);
+aeNoNext(ski1, "r1", "foo:cq1", 3, "");
+ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET,
false);
+ae(ski1, "r1", "foo:cq1", 3, "");
+assertFalse(ski1.hasTop());
+
+ski1.close();
+ }
+
+ @Test
public void test5() throws Exception {
InMemoryMap imm = newInMemoryMap(false,
tempFolder.newFolder().getAbsolutePath());
[2/2] accumulo git commit: Merge branch '1.8'
Merge branch '1.8' Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/016ea678 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/016ea678 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/016ea678 Branch: refs/heads/master Commit: 016ea678033076e34d66f8c0d5a3c1990f8b3df8 Parents: 551fa09 83fe980 Author: Drew Farris <d...@apache.org> Authored: Tue Oct 11 20:42:28 2016 -0400 Committer: Drew Farris <d...@apache.org> Committed: Tue Oct 11 20:42:28 2016 -0400 -- .../accumulo/tserver/InMemoryMapTest.java | 26 1 file changed, 26 insertions(+) --
[1/2] accumulo git commit: ACCUMULO-4483 Added another test case
Repository: accumulo
Updated Branches:
refs/heads/master 551fa0954 -> 016ea6780
ACCUMULO-4483 Added another test case
This forces the exception by reseeking the in memory map
iterator to the same location after forcing a switch.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/83fe980f
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/83fe980f
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/83fe980f
Branch: refs/heads/master
Commit: 83fe980f43864a951f5aa5222ab7428a0d4853ae
Parents: 69ca25f
Author: Ivan Bella <i...@bella.name>
Authored: Tue Oct 11 20:20:10 2016 -0400
Committer: Drew Farris <d...@apache.org>
Committed: Tue Oct 11 20:40:33 2016 -0400
--
.../accumulo/tserver/InMemoryMapTest.java | 26
1 file changed, 26 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/83fe980f/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
--
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
index 59dc68e..f813d42 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/InMemoryMapTest.java
@@ -141,6 +141,13 @@ public class InMemoryMapTest {
}
+ static void aeNoNext(SortedKeyValueIterator<Key,Value> dc, String row,
String column, int ts, String val) throws IOException {
+assertTrue(dc.hasTop());
+assertEquals(nk(row, column, ts), dc.getTopKey());
+assertEquals(new Value(val.getBytes()), dc.getTopValue());
+
+ }
+
static Set newCFSet(String... cfs) {
HashSet cfSet = new HashSet<>();
for (String cf : cfs) {
@@ -248,6 +255,25 @@ public class InMemoryMapTest {
}
@Test
+ public void testDecodeValueModification() throws Exception {
+// This test case is the fix for ACCUMULO-4483
+InMemoryMap imm = newInMemoryMap(false,
tempFolder.newFolder().getAbsolutePath());
+
+mutate(imm, "r1", "foo:cq1", 3, "");
+MemoryIterator ski1 = imm.skvIterator(null);
+
+imm.delete(0);
+
+ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET,
false);
+aeNoNext(ski1, "r1", "foo:cq1", 3, "");
+ski1.seek(new Range(new Text("r1")), LocalityGroupUtil.EMPTY_CF_SET,
false);
+ae(ski1, "r1", "foo:cq1", 3, "");
+assertFalse(ski1.hasTop());
+
+ski1.close();
+ }
+
+ @Test
public void test5() throws Exception {
InMemoryMap imm = newInMemoryMap(false,
tempFolder.newFolder().getAbsolutePath());
svn commit: r930344 - in /websites/production/accumulo/content: ./ 1.4/apidocs/ 1.5/apidocs/ 1.6/apidocs/
Author: drew Date: Mon Nov 24 21:08:53 2014 New Revision: 930344 Log: Publishing svnmucc operation to accumulo site by drew Added: websites/production/accumulo/content/ - copied from r930319, websites/staging/accumulo/trunk/content/ websites/production/accumulo/content/1.4/apidocs/ - copied from r930343, websites/production/accumulo/content/1.4/apidocs/ websites/production/accumulo/content/1.5/apidocs/ - copied from r930343, websites/production/accumulo/content/1.5/apidocs/ websites/production/accumulo/content/1.6/apidocs/ - copied from r930343, websites/production/accumulo/content/1.6/apidocs/
svn commit: r1641305 - in /accumulo/site/trunk/content: papers.mdtext papers/ papers/accumulo-benchmarking-2.1.pdf
Author: drew Date: Mon Nov 24 01:55:33 2014 New Revision: 1641305 URL: http://svn.apache.org/r1641305 Log: Added updated Accumulo Benchmarking paper to the papers page. Added: accumulo/site/trunk/content/papers/ accumulo/site/trunk/content/papers/accumulo-benchmarking-2.1.pdf (with props) Modified: accumulo/site/trunk/content/papers.mdtext Modified: accumulo/site/trunk/content/papers.mdtext URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/papers.mdtext?rev=1641305r1=1641304r2=1641305view=diff == --- accumulo/site/trunk/content/papers.mdtext (original) +++ accumulo/site/trunk/content/papers.mdtext Mon Nov 24 01:55:33 2014 @@ -50,13 +50,20 @@ Notice:Licensed to the Apache Softwa tdYes/td td/td /tr -trtdBenchmarking Apache Accumulo BigData Distributed Table Store Using Its Continuous Test Suite - Sen et al 2013 IEEE International Congress on Big Data/td +trtdBenchmarking Apache Accumulo BigData Distributed Table Store Using Its Continuous Test Suite - Sen et al 2013 IEEE International Congress on Big Data./td tda href=http://sqrrl.com/media/Accumulo-Benchmark-10312013-1.pdf;paper/a/td tdPerformance/td tdIEEE International Congress on Big Data 2013/td tdYes/td td/td /tr +trtdBenchmarking the Apache Accumulo Distributed Key-Value Store - Sen et al 2014 Updated version of the IEEE paper, including edits for clarity and additional results from benchmarking on Amazon EC2./i/td + tda href=papers/accumulo-benchmarking-2.1.pdfpaper/a/td + tdPerformance/td + tdOnline/td + tdNo/td + td/td +/tr trtdComputing on Masked Data: a High Performance Method for Improving Big Data Veracity - Kepner et al IEEE HPEC 2014/td tda href=http://arxiv.org/abs/1406.5751;paper/a/td tdSecurity/td Added: accumulo/site/trunk/content/papers/accumulo-benchmarking-2.1.pdf URL: http://svn.apache.org/viewvc/accumulo/site/trunk/content/papers/accumulo-benchmarking-2.1.pdf?rev=1641305view=auto == Binary file - no diff available. Propchange: accumulo/site/trunk/content/papers/accumulo-benchmarking-2.1.pdf -- svn:mime-type = application/octet-stream
git commit: ACCUMULO-2803 Agitator: Independent Tablet Server and Datanode Min/Max Kill Values
Repository: accumulo
Updated Branches:
refs/heads/1.5.2-SNAPSHOT affb9743a - c839e2ff7
ACCUMULO-2803 Agitator: Independent Tablet Server and Datanode Min/Max Kill
Values
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c839e2ff
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c839e2ff
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c839e2ff
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: c839e2ff7491685be07c3461f9b5ead347ba
Parents: affb974
Author: Drew Farris d...@apache.org
Authored: Fri Jun 6 18:17:13 2014 -0700
Committer: Drew Farris d...@apache.org
Committed: Fri Jun 6 18:17:13 2014 -0700
--
.../system/continuous/continuous-env.sh.example | 30 ++--
test/system/continuous/start-agitator.sh| 12
2 files changed, 27 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c839e2ff/test/system/continuous/continuous-env.sh.example
--
diff --git a/test/system/continuous/continuous-env.sh.example
b/test/system/continuous/continuous-env.sh.example
index bf4e99e..d3fdd62 100644
--- a/test/system/continuous/continuous-env.sh.example
+++ b/test/system/continuous/continuous-env.sh.example
@@ -74,18 +74,30 @@ SLEEP_TIME=10
#set auths.
AUTHS=''
-#time amount of time (in minutes) the agitator should sleep before killing
-KILL_SLEEP_TIME=20
-
-#time amount of time (in minutes) the agitator should sleep after killing
before running tup
-TUP_SLEEP_TIME=10
-
#determines if checksum are generated, may want to turn of when performance
testing
CHECKSUM=true
-#the minimum and maximum server the agitator will kill at once
-MIN_KILL=1
-MAX_KILL=1
+#the amount of time (in minutes) the agitator should sleep before killing
tservers
+TSERVER_KILL_SLEEP_TIME=20
+
+#the amount of time (in minutes) the agitator should sleep after killing
+# before restarting tservers
+TSERVER_RESTART_SLEEP_TIME=10
+
+#the minimum and maximum number of tservers the agitator will kill at once
+TSERVER_MIN_KILL=1
+TSERVER_MAX_KILL=1
+
+#the amount of time (in minutes) the agitator should sleep before killing
datanodes
+DATANODE_KILL_SLEEP_TIME=20
+
+#the amount of time (in minutes) the agitator should sleep after killing
+# before restarting datanodes
+DATANODE_RESTART_SLEEP_TIME=10
+
+#the minimum and maximum number of datanodes the agitator will kill at once
+DATANODE_MIN_KILL=1
+DATANODE_MAX_KILL=1
#time in minutes between killing masters
MASTER_KILL_SLEEP_TIME=60
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c839e2ff/test/system/continuous/start-agitator.sh
--
diff --git a/test/system/continuous/start-agitator.sh
b/test/system/continuous/start-agitator.sh
index 70d5c53..e82e5ab 100755
--- a/test/system/continuous/start-agitator.sh
+++ b/test/system/continuous/start-agitator.sh
@@ -40,18 +40,18 @@ if [[ `whoami` == root ]]; then
# Change to the correct user if started as root
su -c nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME
$MASTER_RESTART_SLEEP_TIME ${LOG_BASE}_master-agitator.out
2${LOG_BASE}_master-agitator.err -m - $ACCUMULO_USER
- su -c nohup ${bin}/tserver-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME
$MIN_KILL $MAX_KILL ${LOG_BASE}_tserver-agitator.out
2${LOG_BASE}_tserver-agitator.err -m - $ACCUMULO_USER
+ su -c nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME
$TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL
${LOG_BASE}_tserver-agitator.out 2${LOG_BASE}_tserver-agitator.err -m -
$ACCUMULO_USER
- su -c nohup ${bin}/datanode-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME
$HADOOP_PREFIX $MIN_KILL $MAX_KILL ${LOG_BASE}_datanode-agitator.out
2${LOG_BASE}_datanode-agitator.err -m - $HDFS_USER
+ su -c nohup ${bin}/datanode-agitator.pl $DATANODE_KILL_SLEEP_TIME
$DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $DATANODE_MIN_KILL
$DATANODE_MAX_KILL ${LOG_BASE}_datanode-agitator.out
2${LOG_BASE}_datanode-agitator.err -m - $HDFS_USER
elif [[ `whoami` == $ACCUMULO_USER ]]; then
echo Running master-agitator and tserver-agitator as `whoami`. Running
datanode-agitator as $HDFS_USER using sudo.
# Just run the master-agitator if we're the accumulo user
nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME
$MASTER_RESTART_SLEEP_TIME ${LOG_BASE}_master-agitator.out
2${LOG_BASE}_master-agitator.err
- nohup ${bin}/tserver-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $MIN_KILL
$MAX_KILL ${LOG_BASE}_tserver-agitator.out 2${LOG_BASE}_tserver-agitator.err
+ nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME
$TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL
${LOG_BASE}_tserver-agitator.out 2
[2/2] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e7e050a6 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e7e050a6 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e7e050a6 Branch: refs/heads/1.6.1-SNAPSHOT Commit: e7e050a6b05b88c90a93405577e7a6eb0751f3c3 Parents: 4950870 c839e2f Author: Drew Farris d...@apache.org Authored: Sat Jun 7 10:11:52 2014 -0400 Committer: Drew Farris d...@apache.org Committed: Sat Jun 7 10:11:52 2014 -0400 -- .../system/continuous/continuous-env.sh.example | 30 ++-- test/system/continuous/start-agitator.sh| 12 2 files changed, 27 insertions(+), 15 deletions(-) --
[1/2] git commit: ACCUMULO-2803 Agitator: Independent Tablet Server and Datanode Min/Max Kill Values
Repository: accumulo
Updated Branches:
refs/heads/1.6.1-SNAPSHOT 4950870de - e7e050a6b
ACCUMULO-2803 Agitator: Independent Tablet Server and Datanode Min/Max Kill
Values
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c839e2ff
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c839e2ff
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c839e2ff
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: c839e2ff7491685be07c3461f9b5ead347ba
Parents: affb974
Author: Drew Farris d...@apache.org
Authored: Fri Jun 6 18:17:13 2014 -0700
Committer: Drew Farris d...@apache.org
Committed: Fri Jun 6 18:17:13 2014 -0700
--
.../system/continuous/continuous-env.sh.example | 30 ++--
test/system/continuous/start-agitator.sh| 12
2 files changed, 27 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c839e2ff/test/system/continuous/continuous-env.sh.example
--
diff --git a/test/system/continuous/continuous-env.sh.example
b/test/system/continuous/continuous-env.sh.example
index bf4e99e..d3fdd62 100644
--- a/test/system/continuous/continuous-env.sh.example
+++ b/test/system/continuous/continuous-env.sh.example
@@ -74,18 +74,30 @@ SLEEP_TIME=10
#set auths.
AUTHS=''
-#time amount of time (in minutes) the agitator should sleep before killing
-KILL_SLEEP_TIME=20
-
-#time amount of time (in minutes) the agitator should sleep after killing
before running tup
-TUP_SLEEP_TIME=10
-
#determines if checksum are generated, may want to turn of when performance
testing
CHECKSUM=true
-#the minimum and maximum server the agitator will kill at once
-MIN_KILL=1
-MAX_KILL=1
+#the amount of time (in minutes) the agitator should sleep before killing
tservers
+TSERVER_KILL_SLEEP_TIME=20
+
+#the amount of time (in minutes) the agitator should sleep after killing
+# before restarting tservers
+TSERVER_RESTART_SLEEP_TIME=10
+
+#the minimum and maximum number of tservers the agitator will kill at once
+TSERVER_MIN_KILL=1
+TSERVER_MAX_KILL=1
+
+#the amount of time (in minutes) the agitator should sleep before killing
datanodes
+DATANODE_KILL_SLEEP_TIME=20
+
+#the amount of time (in minutes) the agitator should sleep after killing
+# before restarting datanodes
+DATANODE_RESTART_SLEEP_TIME=10
+
+#the minimum and maximum number of datanodes the agitator will kill at once
+DATANODE_MIN_KILL=1
+DATANODE_MAX_KILL=1
#time in minutes between killing masters
MASTER_KILL_SLEEP_TIME=60
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c839e2ff/test/system/continuous/start-agitator.sh
--
diff --git a/test/system/continuous/start-agitator.sh
b/test/system/continuous/start-agitator.sh
index 70d5c53..e82e5ab 100755
--- a/test/system/continuous/start-agitator.sh
+++ b/test/system/continuous/start-agitator.sh
@@ -40,18 +40,18 @@ if [[ `whoami` == root ]]; then
# Change to the correct user if started as root
su -c nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME
$MASTER_RESTART_SLEEP_TIME ${LOG_BASE}_master-agitator.out
2${LOG_BASE}_master-agitator.err -m - $ACCUMULO_USER
- su -c nohup ${bin}/tserver-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME
$MIN_KILL $MAX_KILL ${LOG_BASE}_tserver-agitator.out
2${LOG_BASE}_tserver-agitator.err -m - $ACCUMULO_USER
+ su -c nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME
$TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL
${LOG_BASE}_tserver-agitator.out 2${LOG_BASE}_tserver-agitator.err -m -
$ACCUMULO_USER
- su -c nohup ${bin}/datanode-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME
$HADOOP_PREFIX $MIN_KILL $MAX_KILL ${LOG_BASE}_datanode-agitator.out
2${LOG_BASE}_datanode-agitator.err -m - $HDFS_USER
+ su -c nohup ${bin}/datanode-agitator.pl $DATANODE_KILL_SLEEP_TIME
$DATANODE_RESTART_SLEEP_TIME $HADOOP_PREFIX $DATANODE_MIN_KILL
$DATANODE_MAX_KILL ${LOG_BASE}_datanode-agitator.out
2${LOG_BASE}_datanode-agitator.err -m - $HDFS_USER
elif [[ `whoami` == $ACCUMULO_USER ]]; then
echo Running master-agitator and tserver-agitator as `whoami`. Running
datanode-agitator as $HDFS_USER using sudo.
# Just run the master-agitator if we're the accumulo user
nohup ${bin}/master-agitator.pl $MASTER_KILL_SLEEP_TIME
$MASTER_RESTART_SLEEP_TIME ${LOG_BASE}_master-agitator.out
2${LOG_BASE}_master-agitator.err
- nohup ${bin}/tserver-agitator.pl $KILL_SLEEP_TIME $TUP_SLEEP_TIME $MIN_KILL
$MAX_KILL ${LOG_BASE}_tserver-agitator.out 2${LOG_BASE}_tserver-agitator.err
+ nohup ${bin}/tserver-agitator.pl $TSERVER_KILL_SLEEP_TIME
$TSERVER_RESTART_SLEEP_TIME $TSERVER_MIN_KILL $TSERVER_MAX_KILL
${LOG_BASE}_tserver-agitator.out 2
[3/3] git commit: Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT
Merge branch '1.5.2-SNAPSHOT' into 1.6.1-SNAPSHOT Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b33cb733 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b33cb733 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b33cb733 Branch: refs/heads/1.6.1-SNAPSHOT Commit: b33cb733b004786f5699194ba990c6ad420e4263 Parents: cc617cd 72fd01f Author: Drew Farris d...@apache.org Authored: Tue May 13 23:03:10 2014 -0400 Committer: Drew Farris d...@apache.org Committed: Tue May 13 23:03:10 2014 -0400 -- bin/generate_monitor_certificate.sh | 4 ++-- test/system/continuous/datanode-agitator.pl | 14 +- 2 files changed, 11 insertions(+), 7 deletions(-) --
[1/2] git commit: ACCUMULO-2790 generate_monitor_certificate.sh fails on OSX
Repository: accumulo Updated Branches: refs/heads/1.5.2-SNAPSHOT 7e9cb850a - 72fd01ff2 ACCUMULO-2790 generate_monitor_certificate.sh fails on OSX Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/312856ae Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/312856ae Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/312856ae Branch: refs/heads/1.5.2-SNAPSHOT Commit: 312856ae6a2a2324f841796a7ddeacbe9c7bca92 Parents: 7e9cb85 Author: Drew Farris d...@apache.org Authored: Tue May 13 22:51:35 2014 -0400 Committer: Drew Farris d...@apache.org Committed: Tue May 13 22:51:35 2014 -0400 -- bin/generate_monitor_certificate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/312856ae/bin/generate_monitor_certificate.sh -- diff --git a/bin/generate_monitor_certificate.sh b/bin/generate_monitor_certificate.sh index d00c811..fe98bb3 100755 --- a/bin/generate_monitor_certificate.sh +++ b/bin/generate_monitor_certificate.sh @@ -28,8 +28,8 @@ bin=$( cd -P $( dirname $SOURCE ) pwd ) . $bin/config.sh ALIAS=default -KEYPASS=$(tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) -STOREPASS=$(tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) +KEYPASS=$(LC_CTYPE=C tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) +STOREPASS=$(LC_CTYPE=C tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) KEYSTOREPATH=$ACCUMULO_HOME/conf/keystore.jks TRUSTSTOREPATH=$ACCUMULO_HOME/conf/cacerts.jks CERTPATH=$ACCUMULO_HOME/conf/server.cer
[2/3] git commit: ACCUMULO-2768 Agitator not restarting all datanodes
ACCUMULO-2768 Agitator not restarting all datanodes
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/72fd01ff
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/72fd01ff
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/72fd01ff
Branch: refs/heads/1.6.1-SNAPSHOT
Commit: 72fd01ff2cbe85b1e9d2908331308a41857591ed
Parents: 312856a
Author: Drew Farris d...@apache.org
Authored: Tue May 13 22:53:48 2014 -0400
Committer: Drew Farris d...@apache.org
Committed: Tue May 13 22:53:48 2014 -0400
--
test/system/continuous/datanode-agitator.pl | 14 +-
1 file changed, 9 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/72fd01ff/test/system/continuous/datanode-agitator.pl
--
diff --git a/test/system/continuous/datanode-agitator.pl
b/test/system/continuous/datanode-agitator.pl
index acb5843..1f469b2 100755
--- a/test/system/continuous/datanode-agitator.pl
+++ b/test/system/continuous/datanode-agitator.pl
@@ -105,7 +105,7 @@ if ($minKill $maxKill){
while(1){
$numToKill = int(rand($maxKill - $minKill + 1)) + $minKill;
- %killed = {};
+ %killed = ();
$server = ;
for($i = 0; $i $numToKill; $i++){
@@ -124,11 +124,15 @@ while(1){
$nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2;
sleep($nextsleep2 * 60);
- $t = strftime %Y%m%d %H:%M:%S, localtime;
- print STDERR $t Starting datanode on $server\n;
- # We can just start as we're the HDFS user
- system(ssh $server '$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode');
+ foreach $restart (keys %killed) {
+
+$t = strftime %Y%m%d %H:%M:%S, localtime;
+
+print STDERR $t Starting datanode on $restart\n;
+# We can just start as we're the HDFS user
+system(ssh $restart '$HADOOP_PREFIX/sbin/hadoop-daemon.sh start
datanode');
+ }
$nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
sleep($nextsleep1 * 60);
[2/2] git commit: ACCUMULO-2768 Agitator not restarting all datanodes
ACCUMULO-2768 Agitator not restarting all datanodes
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/72fd01ff
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/72fd01ff
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/72fd01ff
Branch: refs/heads/1.5.2-SNAPSHOT
Commit: 72fd01ff2cbe85b1e9d2908331308a41857591ed
Parents: 312856a
Author: Drew Farris d...@apache.org
Authored: Tue May 13 22:53:48 2014 -0400
Committer: Drew Farris d...@apache.org
Committed: Tue May 13 22:53:48 2014 -0400
--
test/system/continuous/datanode-agitator.pl | 14 +-
1 file changed, 9 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/accumulo/blob/72fd01ff/test/system/continuous/datanode-agitator.pl
--
diff --git a/test/system/continuous/datanode-agitator.pl
b/test/system/continuous/datanode-agitator.pl
index acb5843..1f469b2 100755
--- a/test/system/continuous/datanode-agitator.pl
+++ b/test/system/continuous/datanode-agitator.pl
@@ -105,7 +105,7 @@ if ($minKill $maxKill){
while(1){
$numToKill = int(rand($maxKill - $minKill + 1)) + $minKill;
- %killed = {};
+ %killed = ();
$server = ;
for($i = 0; $i $numToKill; $i++){
@@ -124,11 +124,15 @@ while(1){
$nextsleep2 = int(rand($sleep2max - $sleep2)) + $sleep2;
sleep($nextsleep2 * 60);
- $t = strftime %Y%m%d %H:%M:%S, localtime;
- print STDERR $t Starting datanode on $server\n;
- # We can just start as we're the HDFS user
- system(ssh $server '$HADOOP_PREFIX/sbin/hadoop-daemon.sh start datanode');
+ foreach $restart (keys %killed) {
+
+$t = strftime %Y%m%d %H:%M:%S, localtime;
+
+print STDERR $t Starting datanode on $restart\n;
+# We can just start as we're the HDFS user
+system(ssh $restart '$HADOOP_PREFIX/sbin/hadoop-daemon.sh start
datanode');
+ }
$nextsleep1 = int(rand($sleep1max - $sleep1)) + $sleep1;
sleep($nextsleep1 * 60);
[1/3] git commit: ACCUMULO-2790 generate_monitor_certificate.sh fails on OSX
Repository: accumulo Updated Branches: refs/heads/1.6.1-SNAPSHOT cc617cdb2 - b33cb733b ACCUMULO-2790 generate_monitor_certificate.sh fails on OSX Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/312856ae Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/312856ae Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/312856ae Branch: refs/heads/1.6.1-SNAPSHOT Commit: 312856ae6a2a2324f841796a7ddeacbe9c7bca92 Parents: 7e9cb85 Author: Drew Farris d...@apache.org Authored: Tue May 13 22:51:35 2014 -0400 Committer: Drew Farris d...@apache.org Committed: Tue May 13 22:51:35 2014 -0400 -- bin/generate_monitor_certificate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/accumulo/blob/312856ae/bin/generate_monitor_certificate.sh -- diff --git a/bin/generate_monitor_certificate.sh b/bin/generate_monitor_certificate.sh index d00c811..fe98bb3 100755 --- a/bin/generate_monitor_certificate.sh +++ b/bin/generate_monitor_certificate.sh @@ -28,8 +28,8 @@ bin=$( cd -P $( dirname $SOURCE ) pwd ) . $bin/config.sh ALIAS=default -KEYPASS=$(tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) -STOREPASS=$(tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) +KEYPASS=$(LC_CTYPE=C tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) +STOREPASS=$(LC_CTYPE=C tr -dc '#-~' /dev/urandom | tr -d '' | head -c 20) KEYSTOREPATH=$ACCUMULO_HOME/conf/keystore.jks TRUSTSTOREPATH=$ACCUMULO_HOME/conf/cacerts.jks CERTPATH=$ACCUMULO_HOME/conf/server.cer
svn commit: r1497486 - in /accumulo/contrib/wikisearch/trunk: ./ ingest/ ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/ query-war/ query/ query/src/test/java/org/apache/accumulo/
Author: drew Date: Thu Jun 27 18:33:01 2013 New Revision: 1497486 URL: http://svn.apache.org/r1497486 Log: Bare minimum to get Wikisearch to compile under 1.5.0-SNAPSHOT and maven-sort-plugin Modified: accumulo/contrib/wikisearch/trunk/ingest/pom.xml accumulo/contrib/wikisearch/trunk/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java accumulo/contrib/wikisearch/trunk/pom.xml accumulo/contrib/wikisearch/trunk/query-war/pom.xml accumulo/contrib/wikisearch/trunk/query/pom.xml accumulo/contrib/wikisearch/trunk/query/src/test/java/org/apache/accumulo/examples/wikisearch/logic/TestQueryLogic.java Modified: accumulo/contrib/wikisearch/trunk/ingest/pom.xml URL: http://svn.apache.org/viewvc/accumulo/contrib/wikisearch/trunk/ingest/pom.xml?rev=1497486r1=1497485r2=1497486view=diff == --- accumulo/contrib/wikisearch/trunk/ingest/pom.xml (original) +++ accumulo/contrib/wikisearch/trunk/ingest/pom.xml Thu Jun 27 18:33:01 2013 @@ -18,53 +18,50 @@ project xmlns=http://maven.apache.org/POM/4.0.0; xmlns:xsi=http://www.w3.org/2001/XMLSchema-instance; xsi:schemaLocation=http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd; modelVersion4.0.0/modelVersion parent -artifactIdaccumulo-wikisearch/artifactId groupIdorg.apache.accumulo/groupId +artifactIdaccumulo-wikisearch/artifactId version1.5.0-SNAPSHOT/version /parent - artifactIdwikisearch-ingest/artifactId namewikisearch-ingest/name - dependencies dependency - groupIdorg.apache.accumulo/groupId - artifactIdaccumulo-core/artifactId + groupIdcom.google.collections/groupId + artifactIdgoogle-collections/artifactId /dependency dependency - groupIdorg.apache.hadoop/groupId - artifactIdhadoop-core/artifactId + groupIdcom.google.protobuf/groupId + artifactIdprotobuf-java/artifactId /dependency dependency - groupIdlog4j/groupId - artifactIdlog4j/artifactId + groupIdcommons-codec/groupId + artifactIdcommons-codec/artifactId /dependency dependency groupIdcommons-lang/groupId artifactIdcommons-lang/artifactId /dependency dependency - groupIdcom.google.collections/groupId - artifactIdgoogle-collections/artifactId + groupIdlog4j/groupId + artifactIdlog4j/artifactId /dependency dependency - groupIdorg.apache.lucene/groupId - artifactIdlucene-core/artifactId + groupIdorg.apache.accumulo/groupId + artifactIdaccumulo-core/artifactId /dependency dependency - groupIdorg.apache.lucene/groupId - artifactIdlucene-wikipedia/artifactId + groupIdorg.apache.hadoop/groupId + artifactIdhadoop-core/artifactId /dependency dependency - groupIdcom.google.protobuf/groupId - artifactIdprotobuf-java/artifactId + groupIdorg.apache.lucene/groupId + artifactIdlucene-core/artifactId /dependency dependency - groupIdcommons-codec/groupId - artifactIdcommons-codec/artifactId + groupIdorg.apache.lucene/groupId + artifactIdlucene-wikipedia/artifactId /dependency /dependencies - build plugins plugin @@ -73,10 +70,10 @@ executions execution idcopy-dependencies/id -phaseprocess-resources/phase goals goalcopy-dependencies/goal /goals +phaseprocess-resources/phase configuration outputDirectorylib/outputDirectory !-- just grab the non-provided runtime dependencies -- @@ -97,5 +94,4 @@ /plugin /plugins /build - -/project \ No newline at end of file +/project Modified: accumulo/contrib/wikisearch/trunk/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java URL: http://svn.apache.org/viewvc/accumulo/contrib/wikisearch/trunk/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java?rev=1497486r1=1497485r2=1497486view=diff == --- accumulo/contrib/wikisearch/trunk/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java (original) +++ accumulo/contrib/wikisearch/trunk/ingest/src/test/java/org/apache/accumulo/examples/wikisearch/reader/AggregatingRecordReaderTest.java Thu Jun 27 18:33:01 2013 @@ -31,12 +31,12 @@ import javax.xml.xpath.XPath; import javax.xml.xpath.XPathExpression; import javax.xml.xpath.XPathFactory; -import org.apache.accumulo.core.util.ContextFactory; import org.apache.accumulo.examples.wikisearch.ingest.WikipediaInputFormat.WikipediaInputSplit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import
svn commit: r1435198 - in /accumulo/branches/1.4/src: core/src/main/java/org/apache/accumulo/core/conf/Property.java server/src/main/java/org/apache/accumulo/server/master/Master.java
Author: drew
Date: Fri Jan 18 15:45:55 2013
New Revision: 1435198
URL: http://svn.apache.org/viewvc?rev=1435198view=rev
Log:
ACCUMULO-778 (Configurable FATE threads in master) backport to 1.4 branch
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1435198r1=1435197r2=1435198view=diff
==
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Fri Jan 18 15:45:55 2013
@@ -72,6 +72,7 @@ public enum Property {
MASTER_BULK_TIMEOUT(master.bulk.timeout, 5m, PropertyType.TIMEDURATION,
The time to wait for a tablet server to process a bulk import request),
MASTER_MINTHREADS(master.server.threads.minimum, 20, PropertyType.COUNT,
The minimum number of threads to use to handle incoming requests.),
MASTER_THREADCHECK(master.server.threadcheck.time, 1s,
PropertyType.TIMEDURATION, The time between adjustments of the server thread
pool.),
+ MASTER_FATE_THREADPOOL_SIZE(master.fate.threadpool.size, 4,
PropertyType.COUNT, The number of threads used to run FAult-Tolerant
Executions. These are primarily table operations like merge.),
// properties that are specific to tablet server behavior
TSERV_PREFIX(tserver., null, PropertyType.PREFIX, Properties in this
category affect the behavior of the tablet servers),
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1435198r1=1435197r2=1435198view=diff
==
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Fri Jan 18 15:45:55 2013
@@ -220,6 +220,7 @@ public class Master implements LiveTServ
private MasterState state = MasterState.INITIAL;
private FateMaster fate;
+ private int fateThreadCount;
volatile private SortedMapTServerInstance,TabletServerStatus tserverStatus
= Collections
.unmodifiableSortedMap(new
TreeMapTServerInstance,TabletServerStatus());
@@ -537,6 +538,7 @@ public class Master implements LiveTServ
log.info(Instance + instance.getInstanceID());
ThriftTransportPool.getInstance().setIdleTime(ServerConfiguration.getSiteConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));
+fateThreadCount =
ServerConfiguration.getSiteConfiguration().getCount(Property.MASTER_FATE_THREADPOOL_SIZE);
hostname = Accumulo.getLocalAddress(args).getHostName();
fs =
TraceFileSystem.wrap(FileUtil.getFileSystem(CachedConfiguration.getInstance(),
ServerConfiguration.getSiteConfiguration()));
@@ -2096,7 +2098,7 @@ public class Master implements LiveTServ
// TODO: add shutdown for fate object
try {
fate = new FateMaster(this, new
org.apache.accumulo.server.fate.ZooStoreMaster(ZooUtil.getRoot(instance) +
Constants.ZFATE,
- ZooReaderWriter.getRetryingInstance()), 4);
+ ZooReaderWriter.getRetryingInstance()), fateThreadCount);
} catch (KeeperException e) {
throw new IOException(e);
} catch (InterruptedException e) {
svn commit: r1435288 - /accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Author: drew
Date: Fri Jan 18 18:33:01 2013
New Revision: 1435288
URL: http://svn.apache.org/viewvc?rev=1435288view=rev
Log:
ACCUMULO-919 backported to 1.4 branch
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1435288r1=1435287r2=1435288view=diff
==
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Fri Jan 18 18:33:01 2013
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.gc;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@@ -92,6 +93,7 @@ import org.apache.commons.cli.ParseExcep
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.Trash;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Logger;
import org.apache.zookeeper.KeeperException;
@@ -111,8 +113,9 @@ public class SimpleGarbageCollector impl
private long gcStartDelay;
private boolean checkForBulkProcessingFiles;
private FileSystem fs;
- private Option optSafeMode, optOffline, optVerboseMode, optAddress;
- private boolean safemode, offline, verbose;
+ private Option optSafeMode, optOffline, optVerboseMode, optAddress,
optNoTrash;
+ private Trash trash = null;
+ private boolean safemode, offline, verbose, noTrash;
private String address;
private CommandLine commandLine;
private ZooLock lock;
@@ -148,10 +151,12 @@ public class SimpleGarbageCollector impl
optOffline = new Option(o, offline, false,
offline mode will run once and check data files directly; this is
dangerous if accumulo is running or not shut down properly);
optAddress = new Option(a, address, true, specify our local address);
+optNoTrash = new Option(t, no-trash, false, do not use hdfs trash if
it is enabled);
opts.addOption(optVerboseMode);
opts.addOption(optSafeMode);
opts.addOption(optOffline);
opts.addOption(optAddress);
+opts.addOption(optNoTrash);
try {
commandLine = new BasicParser().parse(opts, args);
@@ -162,6 +167,7 @@ public class SimpleGarbageCollector impl
offline = commandLine.hasOption(optOffline.getOpt());
verbose = commandLine.hasOption(optVerboseMode.getOpt());
address = commandLine.getOptionValue(optAddress.getOpt());
+ noTrash = commandLine.hasOption(optNoTrash.getOpt());
} catch (ParseException e) {
String str = Can't parse the command line options;
log.fatal(str, e);
@@ -169,7 +175,7 @@ public class SimpleGarbageCollector impl
}
}
- public void init(FileSystem fs, Instance instance, AuthInfo credentials,
AccumuloConfiguration conf) {
+ public void init(FileSystem fs, Instance instance, AuthInfo credentials,
AccumuloConfiguration conf) throws IOException {
this.fs = fs;
this.instance = instance;
this.credentials = credentials;
@@ -182,8 +188,12 @@ public class SimpleGarbageCollector impl
log.info(safemode: + safemode);
log.info(offline: + offline);
log.info(verbose: + verbose);
+log.info(trash enabled: + !noTrash);
log.info(memory threshold: + CANDIDATE_MEMORY_PERCENTAGE + of +
Runtime.getRuntime().maxMemory() + bytes);
log.info(delete threads: + numDeleteThreads);
+if (!noTrash) {
+ this.trash = new Trash(fs, fs.getConf());
+}
}
private void run() {
@@ -303,6 +313,16 @@ public class SimpleGarbageCollector impl
}
}
+ private boolean moveToTrash(Path path) throws IOException {
+if (trash == null)
+ return false;
+try {
+ return trash.moveToTrash(path);
+} catch (FileNotFoundException ex) {
+ return false;
+}
+ }
+
/*
* this method removes deleted table dirs that are empty
*/
@@ -333,8 +353,11 @@ public class SimpleGarbageCollector impl
if (tabletDirs == null)
continue;
- if (tabletDirs.length == 0)
-fs.delete(new Path(ServerConstants.getTablesDir() + / + delTableId),
false);
+ if (tabletDirs.length == 0) {
+Path p = new Path(ServerConstants.getTablesDir() + / + delTableId);
+if (!moveToTrash(p))
+ fs.delete(p, false);
+ }
}
}
@@ -583,7 +606,7 @@ public class SimpleGarbageCollector impl
Path p = new Path(ServerConstants.getTablesDir() + delete);
-if (fs.delete
svn commit: r1415913 - in /accumulo/branches/1.4.2/docs/examples: README.bulkIngest README.filedata README.mapred
Author: drew Date: Sat Dec 1 04:36:57 2012 New Revision: 1415913 URL: http://svn.apache.org/viewvc?rev=1415913view=rev Log: ACCUMULO-889 update jar regex im examples, scripts Modified: accumulo/branches/1.4.2/docs/examples/README.bulkIngest accumulo/branches/1.4.2/docs/examples/README.filedata accumulo/branches/1.4.2/docs/examples/README.mapred Modified: accumulo/branches/1.4.2/docs/examples/README.bulkIngest URL: http://svn.apache.org/viewvc/accumulo/branches/1.4.2/docs/examples/README.bulkIngest?rev=1415913r1=1415912r2=1415913view=diff == --- accumulo/branches/1.4.2/docs/examples/README.bulkIngest (original) +++ accumulo/branches/1.4.2/docs/examples/README.bulkIngest Sat Dec 1 04:36:57 2012 @@ -28,7 +28,7 @@ accumulo instance name, and a comma-sepa $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.SetupTable instance zookeepers username password test_bulk row_0333 row_0666 $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.GenerateTestData 0 1000 bulk/test_1.txt -$ ./bin/tool.sh lib/examples-simple-*[^c].jar org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample instance zookeepers username password test_bulk bulk tmp/bulkWork +$ ./bin/tool.sh lib/examples-simple-*[^cs].jar org.apache.accumulo.examples.simple.mapreduce.bulk.BulkIngestExample instance zookeepers username password test_bulk bulk tmp/bulkWork $ ./bin/accumulo org.apache.accumulo.examples.simple.mapreduce.bulk.VerifyIngest instance zookeepers username password test_bulk 0 1000 For a high level discussion of bulk ingest, see the docs dir. Modified: accumulo/branches/1.4.2/docs/examples/README.filedata URL: http://svn.apache.org/viewvc/accumulo/branches/1.4.2/docs/examples/README.filedata?rev=1415913r1=1415912r2=1415913view=diff == --- accumulo/branches/1.4.2/docs/examples/README.filedata (original) +++ accumulo/branches/1.4.2/docs/examples/README.filedata Sat Dec 1 04:36:57 2012 @@ -40,7 +40,7 @@ Open the accumulo shell and look at the Run the CharacterHistogram MapReduce to add some information about the file. -$ bin/tool.sh lib/examples-simple*[^c].jar org.apache.accumulo.examples.simple.filedata.CharacterHistogram instance zookeepers username password dataTable exampleVis exampleVis +$ bin/tool.sh lib/examples-simple*[^cs].jar org.apache.accumulo.examples.simple.filedata.CharacterHistogram instance zookeepers username password dataTable exampleVis exampleVis Scan again to see the histogram stored in the 'info' column family. Modified: accumulo/branches/1.4.2/docs/examples/README.mapred URL: http://svn.apache.org/viewvc/accumulo/branches/1.4.2/docs/examples/README.mapred?rev=1415913r1=1415912r2=1415913view=diff == --- accumulo/branches/1.4.2/docs/examples/README.mapred (original) +++ accumulo/branches/1.4.2/docs/examples/README.mapred Sat Dec 1 04:36:57 2012 @@ -50,7 +50,7 @@ for the column family count. After creating the table, run the word count map reduce job. -$ bin/tool.sh lib/examples-simple*[^c].jar org.apache.accumulo.examples.simple.mapreduce.WordCount instance zookeepers /user/username/wc wordCount -u username -p password +$ bin/tool.sh lib/examples-simple*[^cs].jar org.apache.accumulo.examples.simple.mapreduce.WordCount instance zookeepers /user/username/wc wordCount -u username -p password 11/02/07 18:20:11 INFO input.FileInputFormat: Total input paths to process : 1 11/02/07 18:20:12 INFO mapred.JobClient: Running job: job_201102071740_0003
