Author: vines
Date: Mon Apr 23 20:15:10 2012
New Revision: 1329420
URL: http://svn.apache.org/viewvc?rev=1329420&view=rev
Log:
ACCUMULO-404 - Tested in multi-node setup, looks good
Modified:
accumulo/branches/1.4/README
accumulo/branches/1.4/docs/config.html
accumulo/branches/1.4/pom.xml
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Modified: accumulo/branches/1.4/README
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/README?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
--- accumulo/branches/1.4/README (original)
+++ accumulo/branches/1.4/README Mon Apr 23 20:15:10 2012
@@ -192,53 +192,42 @@ certain column.
If you are running on top of hdfs with kerberos enabled, then you need to do
-some extra work. We currently do not internally support kerberos, so you must
-manually manage the accumulo users tickets. First, create an accumulo principal
+some extra work. First, create an Accumulo principal
kadmin.local -q "addprinc -randkey accumulo/<host.domain.name>"
where <host.domain.name> is replaced by a fully qualified domain name. Export
-the principals to a keytab file
+the principals to a keytab file. It is safer to create a unique keytab file
for each
+server, but you can also glob them if you wish.
kadmin.local -q "xst -k accumulo.keytab -glob accumulo*"
Place this file in $ACCUMULO_HOME/conf for every host. It should be owned by
the accumulo user and chmodded to 400. Add the following to the accumulo-env.sh
- kinit -kt $ACCUMULO_HOME/conf/accumulo.keytab accumulo/`hostname -f`
-
-And set the following crontab for every host
-
- 0 5 * * * kinit -kt $ACCUMULO_HOME/conf/accumulo.keytab accumulo/`hostname
-f`
-
-Additionally, adjust the $ACCUMULO_HOME/conf/monitor.security.policy to change
-
- permission java.util.PropertyPermission "*", "read";
-
-to
-
- permission java.util.PropertyPermission "*", "read,write";
-
-And add these lines to the end of the policy file
-
- permission javax.security.auth.AuthPermission
"createLoginContext.hadoop-user-kerberos";
- permission java.lang.RuntimePermission "createSecurityManager";
- permission javax.security.auth.AuthPermission "doAs";
- permission javax.security.auth.AuthPermission "getPolicy";
- permission java.security.SecurityPermission "createAccessControlContext";
- permission javax.security.auth.AuthPermission "getSubjectFromDomainCombiner";
- permission java.lang.RuntimePermission "getProtectionDomain";
- permission javax.security.auth.AuthPermission "modifyPrivateCredentials";
- permission javax.security.auth.PrivateCredentialPermission
"javax.security.auth.kerberos.KerberosTicket
javax.security.auth.kerberos.KerberosPrincipal \"*\"", "read";
- permission javax.security.auth.kerberos.ServicePermission
"krbtgt/<REALM>@<REALM>", "initiate";
- permission javax.security.auth.kerberos.ServicePermission
"hdfs/<namenode.domain.name>@<REALM>", "initiate";
- permission javax.security.auth.kerberos.ServicePermission
"mapred/<jobtracker.domain.name>@<REALM>", "initiate";
-
-Where <REALM> is replaced with the kerberos realm for the Hadoop cluster,
-<namenode.domain.name> is replaced with the fully qualified domain name of the
-server running the namenode and <jobtracker.domain.name> is replaced with the
-fully qualified domain name of the server running the job tracker.
-
+In the accumulo-site.xml file on each node, add settings for
general.kerberos.keytab
+and general.kerberos.principal, where the keytab setting is the absolute path
+to the keytab file ($ACCUMULO_HOME is valid to use) and principal is set to
+accumulo/_HOST@<REALM>, where REALM is set to your kerberos realm. You may use
+_HOST in lieu of your individual host names.
+
+ <property>
+ <name>general.kerberos.keytab</name>
+ <value>$ACCUMULO_HOME/conf/accumulo.keytab</value>
+ </property>
+
+ <property>
+ <name>general.kerberos.principal</name>
+ <value>accumulo/_HOST@MYREALM</value>
+ </property>
+
+You can then start up Accumulo as you would with the accumulo user, and it will
+automatically handle the kerberos keys needed to access hdfs.
+
+Please Note: You may have issues initializing Accumulo while running kerberos
HDFS.
+You can resolve this by temporarily granting the accumulo user write access to
the
+hdfs root directory, running init, and then revoking write permission in the
root
+directory (be sure to maintain access to the /accumulo directory).
******************************************************************************
6. Monitoring Apache Accumulo
Modified: accumulo/branches/1.4/docs/config.html
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/docs/config.html?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
--- accumulo/branches/1.4/docs/config.html (original)
+++ accumulo/branches/1.4/docs/config.html Mon Apr 23 20:15:10 2012
@@ -155,6 +155,20 @@ $HADOOP_HOME/lib/[^.].*.jar,
<td>A list of all of the places where changes in jars or classes will
force a reload of the classloader.</td>
</tr>
<tr class='highlight'>
+ <td>general.kerberos.keytab</td>
+ <td><b><a href='#PATH'>path</a></b></td>
+ <td>no</td>
+ <td><pre> </pre></td>
+ <td>Path to the kerberos keytab to use. Leave blank if not using
kerberoized hdfs</td>
+ </tr>
+ <tr >
+ <td>general.kerberos.principal</td>
+ <td><b><a href='#STRING'>string</a></b></td>
+ <td>no</td>
+ <td><pre> </pre></td>
+ <td>Name of the kerberos principal to use. _HOST will automatically be
replaced by the machines hostname in the hostname portion of the principal.
Leave blank if not using kerberoized hdfs</td>
+ </tr>
+ <tr class='highlight'>
<td>general.rpc.timeout</td>
<td><b><a href='#TIMEDURATION'>duration</a></b></td>
<td>no</td>
Modified: accumulo/branches/1.4/pom.xml
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/pom.xml?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
--- accumulo/branches/1.4/pom.xml (original)
+++ accumulo/branches/1.4/pom.xml Mon Apr 23 20:15:10 2012
@@ -636,7 +636,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
- <version>0.20.2</version>
+ <version>0.20.203.0</version>
<scope>provided</scope>
</dependency>
<dependency>
Modified:
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
(original)
+++
accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/conf/Property.java
Mon Apr 23 20:15:10 2012
@@ -45,11 +45,13 @@ public enum Property {
GENERAL_CLASSPATHS(AccumuloClassLoader.CLASSPATH_PROPERTY_NAME,
AccumuloClassLoader.DEFAULT_CLASSPATH_VALUE, PropertyType.STRING,
"A list of all of the places to look for a class. Order does matter, as
it will look for the jar "
+ "starting in the first location to the last. Please note, hadoop
conf and hadoop lib directories NEED to be here, "
- + "along with accumulo lib and zookeeper directory. Supports full
regex on filename alone."), // needs special treatment in accumulo start
-
// jar
+ + "along with accumulo lib and zookeeper directory. Supports full
regex on filename alone."), // needs special treatment in accumulo start jar
GENERAL_DYNAMIC_CLASSPATHS(AccumuloClassLoader.DYNAMIC_CLASSPATH_PROPERTY_NAME,
AccumuloClassLoader.DEFAULT_DYNAMIC_CLASSPATH_VALUE, PropertyType.STRING,
"A list of all of the places where changes in jars or classes will force
a reload of the classloader."),
GENERAL_RPC_TIMEOUT("general.rpc.timeout", "120s",
PropertyType.TIMEDURATION, "Time to wait on I/O for simple, short RPC calls"),
+ GENERAL_KERBEROS_KEYTAB("general.kerberos.keytab", "", PropertyType.PATH,
"Path to the kerberos keytab to use. Leave blank if not using kerberoized
hdfs"),
+ GENERAL_KERBEROS_PRINCIPAL("general.kerberos.principal", "",
PropertyType.STRING, "Name of the kerberos principal to use. _HOST will
automatically be "
+ + "replaced by the machines hostname in the hostname portion of the
principal. Leave blank if not using kerberoized hdfs"),
// properties that are specific to master server behavior
MASTER_PREFIX("master.", null, PropertyType.PREFIX, "Properties in this
category affect the behavior of the master server"),
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/gc/SimpleGarbageCollector.java
Mon Apr 23 20:15:10 2012
@@ -75,6 +75,7 @@ import org.apache.accumulo.server.client
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.state.tables.TableManager;
import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.accumulo.server.util.Halt;
import org.apache.accumulo.server.util.OfflineMetadataScanner;
@@ -122,6 +123,8 @@ public class SimpleGarbageCollector impl
private int numDeleteThreads;
public static void main(String[] args) throws UnknownHostException,
IOException {
+ SecurityUtil.serverLogin();
+
Accumulo.init("gc");
SimpleGarbageCollector gc = new SimpleGarbageCollector(args);
@@ -185,7 +188,7 @@ public class SimpleGarbageCollector impl
private void run() {
long tStart, tStop;
-
+
// Sleep for an initial period, giving the master time to start up and
// old data files to be unused
if (!offline) {
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/logger/LogService.java
Mon Apr 23 20:15:10 2012
@@ -62,6 +62,7 @@ import org.apache.accumulo.server.client
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.logger.LogWriter.LogWriteException;
import org.apache.accumulo.server.security.Authenticator;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.security.ZKAuthenticator;
import org.apache.accumulo.server.trace.TraceFileSystem;
import org.apache.accumulo.server.util.FileSystemMonitor;
@@ -121,7 +122,8 @@ public class LogService implements Mutat
public static void main(String[] args) throws Exception {
LogService logService;
-
+ SecurityUtil.serverLogin();
+
try {
logService = new LogService(args);
} catch (Exception e) {
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/Master.java
Mon Apr 23 20:15:10 2012
@@ -141,6 +141,7 @@ import org.apache.accumulo.server.master
import org.apache.accumulo.server.monitor.Monitor;
import org.apache.accumulo.server.security.Authenticator;
import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.security.ZKAuthenticator;
import org.apache.accumulo.server.tabletserver.TabletTime;
import org.apache.accumulo.server.tabletserver.log.RemoteLogger;
@@ -529,7 +530,6 @@ public class Master implements LiveTServ
}
public Master(String[] args) throws IOException {
-
Accumulo.init("master");
log.info("Version " + Constants.VERSION);
@@ -2151,6 +2151,8 @@ public class Master implements LiveTServ
public static void main(String[] args) throws Exception {
try {
+ SecurityUtil.serverLogin();
+
Master master = new Master(args);
master.run();
} catch (Exception ex) {
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/master/state/SetGoalState.java
Mon Apr 23 20:15:10 2012
@@ -22,6 +22,7 @@ import org.apache.accumulo.core.zookeepe
import org.apache.accumulo.core.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
public class SetGoalState {
@@ -34,6 +35,8 @@ public class SetGoalState {
System.err.println("Usage: accumulo " + SetGoalState.class.getName() + "
[NORMAL|SAFE_MODE|CLEAN_STOP]");
System.exit(-1);
}
+ SecurityUtil.serverLogin();
+
Accumulo.waitForZookeeperAndHdfs();
ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(HdfsZooInstance.getInstance())
+ Constants.ZMASTER_GOAL_STATE, args[0].getBytes(),
NodeExistsPolicy.OVERWRITE);
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/monitor/Monitor.java
Mon Apr 23 20:15:10 2012
@@ -67,6 +67,7 @@ import org.apache.accumulo.server.monito
import org.apache.accumulo.server.problems.ProblemReports;
import org.apache.accumulo.server.problems.ProblemType;
import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.util.EmbeddedWebServer;
import org.apache.log4j.Logger;
import org.apache.zookeeper.WatchedEvent;
@@ -422,6 +423,7 @@ public class Monitor {
}
public static void main(String[] args) {
+ SecurityUtil.serverLogin();
new Monitor().run(args);
}
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/tabletserver/TabletServer.java
Mon Apr 23 20:15:10 2012
@@ -145,6 +145,7 @@ import org.apache.accumulo.server.proble
import org.apache.accumulo.server.problems.ProblemReports;
import org.apache.accumulo.server.security.Authenticator;
import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.security.ZKAuthenticator;
import org.apache.accumulo.server.tabletserver.Tablet.CommitSession;
import org.apache.accumulo.server.tabletserver.Tablet.KVEntry;
@@ -223,6 +224,8 @@ public class TabletServer extends Abstra
public TabletServer() {
super();
+ watcher = new TransactionWatcher();
+
SimpleTimer.getInstance().schedule(new TimerTask() {
@Override
public void run() {
@@ -810,7 +813,7 @@ public class TabletServer extends Abstra
}
}
- TransactionWatcher watcher = new TransactionWatcher();
+ TransactionWatcher watcher;
private class ThriftClientHandler extends ClientServiceHandler implements
TabletClientService.Iface {
@@ -2651,6 +2654,8 @@ public class TabletServer extends Abstra
// main loop listens for client requests
public void run() {
+ SecurityUtil.serverLogin();
+
int clientPort = 0;
try {
clientPort = startTabletClientService();
@@ -3102,6 +3107,8 @@ public class TabletServer extends Abstra
public static void main(String[] args) throws IOException {
try {
+ SecurityUtil.serverLogin();
+
TabletServer server = new TabletServer();
server.config(args);
server.run();
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/trace/TraceServer.java
Mon Apr 23 20:15:10 2012
@@ -39,6 +39,7 @@ import org.apache.accumulo.core.zookeepe
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.util.time.SimpleTimer;
import org.apache.accumulo.server.zookeeper.IZooReaderWriter;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
@@ -219,6 +220,7 @@ public class TraceServer implements Watc
}
public static void main(String[] args) throws Exception {
+ SecurityUtil.serverLogin();
TraceServer server = new TraceServer(args);
server.run();
log.info("tracer stopping");
Modified:
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
URL:
http://svn.apache.org/viewvc/accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java?rev=1329420&r1=1329419&r2=1329420&view=diff
==============================================================================
---
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
(original)
+++
accumulo/branches/1.4/src/server/src/main/java/org/apache/accumulo/server/util/Initialize.java
Mon Apr 23 20:15:10 2012
@@ -49,6 +49,7 @@ import org.apache.accumulo.server.constr
import org.apache.accumulo.server.iterators.MetadataBulkLoadFilter;
import org.apache.accumulo.server.master.state.tables.TableManager;
import org.apache.accumulo.server.security.SecurityConstants;
+import org.apache.accumulo.server.security.SecurityUtil;
import org.apache.accumulo.server.security.ZKAuthenticator;
import org.apache.accumulo.server.tabletserver.TabletTime;
import org.apache.accumulo.server.zookeeper.IZooReaderWriter;
@@ -423,7 +424,10 @@ public class Initialize {
try {
Configuration conf = CachedConfiguration.getInstance();
+ SecurityUtil.serverLogin();
+
FileSystem fs = FileUtil.getFileSystem(conf,
ServerConfiguration.getSiteConfiguration());
+
if (justSecurity) {
if (isInitialized(fs))
initSecurity(HdfsZooInstance.getInstance().getInstanceID(),
getRootPassword());
