svn commit: r960079 - in /hadoop/common/trunk: CHANGES.txt src/java/core-default.xml src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java src/java/org/apache/hadoop/fs/CommonConfigurationKeysPub
Author: shv Date: Fri Jul 2 18:16:30 2010 New Revision: 960079 URL: http://svn.apache.org/viewvc?rev=960079view=rev Log: HADOOP-6756. Documentation for common configuration keys. Contributed by Erik Steffl. Added: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java (with props) Modified: hadoop/common/trunk/CHANGES.txt hadoop/common/trunk/src/java/core-default.xml hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Modified: hadoop/common/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=960079r1=960078r2=960079view=diff == --- hadoop/common/trunk/CHANGES.txt (original) +++ hadoop/common/trunk/CHANGES.txt Fri Jul 2 18:16:30 2010 @@ -2,15 +2,19 @@ Hadoop Change Log Trunk (unreleased changes) + INCOMPATIBLE CHANGES + NEW FEATURES - HADOOP-6791. Refresh for proxy superuser config + +HADOOP-6791. Refresh for proxy superuser config (common part for HDFS-1096) (boryas) - HADOOP-6581. Add authenticated TokenIdentifiers to UGI so that - they can be used for authorization (Kan Zhang and Jitendra Pandey - via jghoman) +HADOOP-6581. Add authenticated TokenIdentifiers to UGI so that +they can be used for authorization (Kan Zhang and Jitendra Pandey +via jghoman) IMPROVEMENTS + HADOOP-6644. util.Shell getGROUPS_FOR_USER_COMMAND method name - should use common naming convention (boryas) @@ -47,7 +51,13 @@ Trunk (unreleased changes) HADOOP-6814. Adds an API in UserGroupInformation to get the real authentication method of a passed UGI. (Jitendra Pandey via ddas) +HADOOP-6756. Documentation for common configuration keys. +(Erik Steffl via shv) + + OPTIMIZATIONS + BUG FIXES + HADOOP-6638. try to relogin in a case of failed RPC connection (expired tgt) only in case the subject is loginUser or proxyUgi.realUser. (boryas) @@ -92,7 +102,7 @@ Trunk (unreleased changes) (ddas) HADOOP-6815. refreshSuperUserGroupsConfiguration should use server side -configuration for the refresh (boryas) +configuration for the refresh (boryas) Release 0.21.0 - Unreleased Modified: hadoop/common/trunk/src/java/core-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/core-default.xml?rev=960079r1=960078r2=960079view=diff == --- hadoop/common/trunk/src/java/core-default.xml (original) +++ hadoop/common/trunk/src/java/core-default.xml Fri Jul 2 18:16:30 2010 @@ -53,6 +53,8 @@ ordering of the filters./description /property +!--- security properties -- + property namehadoop.security.authorization/name valuefalse/value @@ -67,6 +69,35 @@ /property property + namehadoop.security.group.mapping/name + valueorg.apache.hadoop.security.ShellBasedUnixGroupsMapping/value + description +Class for user to group mapping (get groups for a given user) for ACL + /description +/property + +property + namehadoop.security.groups.cache.secs/name + value300/value + description +This is the config controlling the validity of the entries in the cache +containing the user-group mapping. When this duration has expired, +then the implementation of the group mapping provider is invoked to get +the groups of the user and then cached back. + /description +/property + +property + namehadoop.security.service.user.name.key/name + value/value + description +For those cases where the same RPC protocol is implemented by multiple +servers, this configuration is required for specifying the principal +name to use for the service when the client wishes to make an RPC call. + /description +/property + +property namehadoop.rpc.protection/name valueauthentication/value descriptionThis field sets the quality of protection for secured sasl @@ -148,6 +179,19 @@ facilitate opening large MapFiles using less memory./description /property +property + nameio.map.index.interval/name + value128/value + description +MapFile consist of two files - data file (tuples) and index file +(keys). For every io.map.index.interval records written in the +data file, an entry (record-key, data-file-position) is written +in the index file. This is to allow for doing binary search later +within the index file to look up records by their keys and get their +closest positions in the data file. + /description +/property + !-- file system properties -- property @@ -240,6 +284,20 @@ /property property + namefs.ftp.host/name + value0.0.0.0/value + descriptionFTP filesystem connects to this server/description +/property + +property + namefs.ftp.host.port/name + value21/value + description +FTP filesystem connects to fs.ftp.host on this port + /description +/property + +property
[Hadoop Wiki] Update of ZooKeeper by PatrickHunt
Dear Wiki user, You have subscribed to a wiki page or wiki category on Hadoop Wiki for change notification. The ZooKeeper page has been changed by PatrickHunt. http://wiki.apache.org/hadoop/ZooKeeper?action=diffrev1=25rev2=26 -- * [[ZooKeeper/ProjectDescription| Overview]] of ZooKeeper * [[ZooKeeper/Tutorial| Tutorial:]] A crash course on how to implement primitives with ZooKeeper * [[ZooKeeper/FAQ| FAQ]] + * [[ZooKeeper/ZKClientBindings| Client bindings]] * [[ZooKeeper/UsefulTools| Useful Tools]] * [[ZooKeeper/ZooKeeperPresentations| Presentations]] and [[ZooKeeper/ZooKeeperArticles| articles]] about ZooKeeper * [[ZooKeeper/PoweredBy| PoweredBy]], a list of sites and applications powered by ZooKeeper
[Hadoop Wiki] Update of topology_rack_awareness_sc ripts by EdwardCapriolo
Dear Wiki user,
You have subscribed to a wiki page or wiki category on Hadoop Wiki for change
notification.
The topology_rack_awareness_scripts page has been changed by EdwardCapriolo.
The comment on this change is: Weird extra characters removed.
http://wiki.apache.org/hadoop/topology_rack_awareness_scripts?action=diffrev1=2rev2=3
--
HADOOP_CONF=/etc/hadoop/conf
while [ $# -gt 0 ] ; do
-
- . nodeArg=$1
+ nodeArg=$1
- exec ${HADOOP_CONF}/topology.data result= while read line ; do
+ exec ${HADOOP_CONF}/topology.data
+ result=
+ while read line ; do
+ ar=( $line )
- . ar=( $line ) if [ ${ar[0]} = $nodeArg ] ; then
+ if [ ${ar[0]} = $nodeArg ] ; then
-. result=${ar[1]}
+ result=${ar[1]}
+ fi
+ done
+ shift if [ -z $result ] ; then
+ echo -n /default-rack
+ else
+ echo -n $result
fi
- done shift if [ -z $result ] ; then
- . echo -n /default-rack
- else
- . echo -n $result
- fi
done
}}}
[Hadoop Wiki] Update of ZooKeeper/ZKClientBindings by PatrickHunt
Dear Wiki user, You have subscribed to a wiki page or wiki category on Hadoop Wiki for change notification. The ZooKeeper/ZKClientBindings page has been changed by PatrickHunt. http://wiki.apache.org/hadoop/ZooKeeper/ZKClientBindings -- New page: ZooKeeper Client Bindings ZooKeeper ships with C, Java, Perl and Python client bindings, here are a list of client bindings that are available from the community but not yet included in the release (we encourage developers to contribute their bindings back to the project - generally we're happy to include as a contrib.) ||Binding||Author||URL|| ||Scala||John Corwin||http://github.com/jcorwin/zookeeper-client|| || || || ||
[Hadoop Wiki] Update of ZooKeeper/ZKClientBindings by PatrickHunt
Dear Wiki user, You have subscribed to a wiki page or wiki category on Hadoop Wiki for change notification. The ZooKeeper/ZKClientBindings page has been changed by PatrickHunt. http://wiki.apache.org/hadoop/ZooKeeper/ZKClientBindings?action=diffrev1=1rev2=2 -- - ZooKeeper Client Bindings + h1. ZooKeeper Client Bindings ZooKeeper ships with C, Java, Perl and Python client bindings, here are a list of client bindings that are available from the community but not yet included in the release (we encourage developers to contribute their bindings back to the project - generally we're happy to include as a contrib.)
[Hadoop Wiki] Update of topology_rack_awareness_sc ripts by EdwardCapriolo
Dear Wiki user,
You have subscribed to a wiki page or wiki category on Hadoop Wiki for change
notification.
The topology_rack_awareness_scripts page has been changed by EdwardCapriolo.
http://wiki.apache.org/hadoop/topology_rack_awareness_scripts?action=diffrev1=3rev2=4
--
result=${ar[1]}
fi
done
+ shift
- shift if [ -z $result ] ; then
+ if [ -z $result ] ; then
echo -n /default-rack
else
echo -n $result
[Hadoop Wiki] Update of ZooKeeper/ZKClientBindings by PatrickHunt
Dear Wiki user, You have subscribed to a wiki page or wiki category on Hadoop Wiki for change notification. The ZooKeeper/ZKClientBindings page has been changed by PatrickHunt. http://wiki.apache.org/hadoop/ZooKeeper/ZKClientBindings?action=diffrev1=3rev2=4 -- - h1. ZooKeeper Client Bindings + == ZooKeeper Client Bindings == ZooKeeper ships with C, Java, Perl and Python client bindings, here are a list of client bindings that are available from the community but not yet included in the release (we encourage developers to contribute their bindings back to the project - generally we're happy to include as a contrib.)
[Hadoop Wiki] Update of ZooKeeper/ZKClientBindings by Eric Hauser
Dear Wiki user, You have subscribed to a wiki page or wiki category on Hadoop Wiki for change notification. The ZooKeeper/ZKClientBindings page has been changed by Eric Hauser. http://wiki.apache.org/hadoop/ZooKeeper/ZKClientBindings?action=diffrev1=4rev2=5 -- ||Binding||Author||URL|| ||Scala||Steve Jenson, John Corwin||http://github.com/twitter/scala-zookeeper-client|| + ||C#||Eric Hauser||http://github.com/ewhauser/zookeeper|| || || || ||
svn commit: r960137 - in /hadoop/common/trunk: CHANGES.txt src/java/org/apache/hadoop/http/HttpServer.java src/java/org/apache/hadoop/security/Krb5AndCertsSslSocketConnector.java
Author: jghoman
Date: Sat Jul 3 00:02:03 2010
New Revision: 960137
URL: http://svn.apache.org/viewvc?rev=960137view=rev
Log:
HADOOP-6584. Provide Kerberized SSL encryption for webservices.
Added:
hadoop/common/trunk/src/java/org/apache/hadoop/security/Krb5AndCertsSslSocketConnector.java
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java
Modified: hadoop/common/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=960137r1=960136r2=960137view=diff
==
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Sat Jul 3 00:02:03 2010
@@ -13,6 +13,9 @@ Trunk (unreleased changes)
they can be used for authorization (Kan Zhang and Jitendra Pandey
via jghoman)
+HADOOP-6584. Provide Kerberized SSL encryption for webservices.
+(jghoman and Kan Zhang via jghoman)
+
IMPROVEMENTS
HADOOP-6644. util.Shell getGROUPS_FOR_USER_COMMAND method name
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java?rev=960137r1=960136r2=960137view=diff
==
--- hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java
(original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java Sat Jul
3 00:02:03 2010
@@ -46,6 +46,8 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.log.LogLevel;
import org.apache.hadoop.metrics.MetricsServlet;
+import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector;
+import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector.MODE;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.ReflectionUtils;
@@ -162,7 +164,11 @@ public class HttpServer implements Filte
webServer.addHandler(webAppContext);
addDefaultApps(contexts, appDir, conf);
-
+
+defineFilter(webAppContext, krb5Filter,
+Krb5AndCertsSslSocketConnector.Krb5SslFilter.class.getName(),
+null, null);
+
addGlobalFilter(safety, QuotingInputFilter.class.getName(), null);
final FilterInitializer[] initializers = getFilterInitializers(conf);
if (initializers != null) {
@@ -290,7 +296,7 @@ public class HttpServer implements Filte
*/
public void addServlet(String name, String pathSpec,
Class? extends HttpServlet clazz) {
-addInternalServlet(name, pathSpec, clazz);
+addInternalServlet(name, pathSpec, clazz, false);
addFilterPathMapping(pathSpec, webAppContext);
}
@@ -306,11 +312,38 @@ public class HttpServer implements Filte
*/
public void addInternalServlet(String name, String pathSpec,
Class? extends HttpServlet clazz) {
+addInternalServlet(name, pathSpec, clazz, false);
+ }
+
+ /**
+ * Add an internal servlet in the server, specifying whether or not to
+ * protect with Kerberos authentication.
+ * Note: This method is to be used for adding servlets that facilitate
+ * internal communication and not for user facing functionality. For
+ * servlets added using this method, filters (except internal Kerberized
+ * filters) are not enabled.
+ *
+ * @param name The name of the servlet (can be passed as null)
+ * @param pathSpec The path spec for the servlet
+ * @param clazz The servlet class
+ */
+ public void addInternalServlet(String name, String pathSpec,
+ Class? extends HttpServlet clazz, boolean requireAuth) {
ServletHolder holder = new ServletHolder(clazz);
if (name != null) {
holder.setName(name);
}
webAppContext.addServlet(holder, pathSpec);
+
+if(requireAuth UserGroupInformation.isSecurityEnabled()) {
+ LOG.info(Adding Kerberos filter to + name);
+ ServletHandler handler = webAppContext.getServletHandler();
+ FilterMapping fmap = new FilterMapping();
+ fmap.setPathSpec(pathSpec);
+ fmap.setFilterName(krb5Filter);
+ fmap.setDispatches(Handler.ALL);
+ handler.addFilterMapping(fmap);
+}
}
/** {...@inheritdoc} */
@@ -451,10 +484,22 @@ public class HttpServer implements Filte
*/
public void addSslListener(InetSocketAddress addr, Configuration sslConf,
boolean needClientAuth) throws IOException {
+addSslListener(addr, sslConf, needClientAuth, false);
+ }
+
+ /**
+ * Configure an ssl listener on the server.
+ * @param addr address to listen on
+ * @param sslConf conf to retrieve ssl options
+ * @param needCertsAuth whether x509 certificate authentication is required
+ * @param needKrbAuth whether to allow kerberos auth
+ */
+ public void
