Author: omalley
Date: Fri Mar 4 04:13:04 2011
New Revision: 1077420
URL: http://svn.apache.org/viewvc?rev=1077420&view=rev
Log:
commit 94e20361420f8fcb65bed1c8a66495d248675c06
Author: Owen O'Malley <omalley@apache.org>
Date: Thu Apr 22 23:32:15 2010 -0700
HADOOP-6670. Use the UserGroupInformation's Subject as the criteria for
equals and hashCode. (omalley)
This reverts commit 6413d7d0b936474ddf355e1e4cc76d16ab773976.
Conflicts:
YAHOO-CHANGES.txt
+++ b/YAHOO-CHANGES.txt
+ HADOOP-6670. Use the UserGroupInformation's Subject as the criteria for
+ equals and hashCode. (omalley)
+
+ task cleanup attempt's log directory. (Amareshwari Sreeramadasu via
+ vinodkv)
+ MAPREDUCE-1533. JobTracker performance enhancements. (Amar Kamat via
+ vinodkv)
+ HDFS-1011. Improve Logging in HDFSProxy to include cluster name associated
+ with the request (Srikanth Sundarrajan via Nicholas)
+ HDFS-1010. Retrieve group information from UnixUserGroupInformation
+ instead of LdapEntry (Srikanth Sundarrajan via Nicholas)
+ HADOOP-6687. user object in the subject in UGI should be reused in case
+ of a relogin. (jitendra)
+ HADOOP-5647. TestJobHistory fails if /tmp/_logs is not writable to.
+ Testcase should not depend on /tmp. (Ravi Gummadi via vinodkv)
+ MAPREDUCE-181. Bug fix for Secure job submission. (Ravi Gummadi via
+ vinodkv)
+ MAPREDUCE-1526. Cache the job related information while submitting the
+ job. (rksingh)
+ HDFS-1005. Fsck security. Makes it work over kerberized SSL (boryas and
+ jhoman)
+ HADOOP-6600,HDFS-1003,MAPREDUCE-1539. mechanism for authorization check
+ for inter-server protocols(boryas)
+
+ HADOOP-6580,HDFS-993,MAPREDUCE-1516. UGI should contain authentication
+ method.
+ Namenode and JT should issue a delegation token only for kerberos
+ authenticated clients. (jitendra)
+ HDFS-984,HADOOP-6573,MAPREDUCE-1537. Delegation Tokens should be persisted
+ in Namenode, and corresponding changes in common and mr. (jitendra)
+ HADOOP-6586. Log authentication and authorization failures and successes
+ (boryas)
+
+x
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java?rev=1077420&r1=1077419&r2=1077420&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/fs/FileSystem.java
Fri Mar 4 04:13:04 2011
@@ -228,7 +228,10 @@ public abstract class FileSystem extends
* @throws IOException
*/
public static void closeAll() throws IOException {
+ LOG.debug("Starting clear of FileSystem cache with " + CACHE.size() +
+ " elements.");
CACHE.closeAll();
+ LOG.debug("Done clearing cache");
}
/** Make sure that a path specifies a FileSystem. */
@@ -1257,6 +1260,7 @@ public abstract class FileSystem extends
// delete all files that were marked as delete-on-exit.
processDeleteOnExit();
CACHE.remove(this.key, this);
+ LOG.debug("Removing filesystem for " + getUri());
}
/** Return the total size of all files in the filesystem.*/
@@ -1382,6 +1386,7 @@ public abstract class FileSystem extends
private static FileSystem createFileSystem(URI uri, Configuration conf
) throws IOException {
Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null);
+ LOG.debug("Creating filesystem for " + uri);
if (clazz == null) {
throw new IOException("No FileSystem for scheme: " + uri.getScheme());
}
@@ -1485,6 +1490,14 @@ public abstract class FileSystem extends
return "("+ugi.toString() + ")@" + scheme + "://" + authority;
}
}
+
+ /**
+ * Get the number of file systems in the cache.
+ * @return the number of cached file systems
+ */
+ int size() {
+ return map.size();
+ }
}
public static final class Statistics {
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java?rev=1077420&r1=1077419&r2=1077420&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/ipc/Client.java
Fri Mar 4 04:13:04 2011
@@ -1050,8 +1050,8 @@ public class Client {
if (obj instanceof ConnectionId) {
ConnectionId id = (ConnectionId) obj;
return address.equals(id.address) && protocol == id.protocol &&
- ticket == id.ticket;
- //Note : ticket is a ref comparision.
+ ((ticket != null && ticket.equals(id.ticket)) ||
+ (ticket == id.ticket));
}
return false;
}
@@ -1059,7 +1059,7 @@ public class Client {
@Override
public int hashCode() {
return (address.hashCode() + PRIME * System.identityHashCode(protocol)) ^
- System.identityHashCode(ticket);
+ (ticket == null ? 0 : ticket.hashCode());
}
}
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1077420&r1=1077419&r2=1077420&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
Fri Mar 4 04:13:04 2011
@@ -935,7 +935,7 @@ public class UserGroupInformation {
} else if (o == null || getClass() != o.getClass()) {
return false;
} else {
- return subject.equals(((UserGroupInformation) o).subject);
+ return subject == ((UserGroupInformation) o).subject;
}
}
@@ -944,7 +944,7 @@ public class UserGroupInformation {
*/
@Override
public int hashCode() {
- return subject.hashCode();
+ return System.identityHashCode(subject);
}
/**
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java?rev=1077420&r1=1077419&r2=1077420&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/filecache/TestTrackerDistributedCacheManager.java
Fri Mar 4 04:13:04 2011
@@ -297,18 +297,57 @@ public class TestTrackerDistributedCache
if (!canRun()) {
return;
}
- checkLocalizedPath("true");
- checkLocalizedPath("false");
+ checkLocalizedPath(true);
+ checkLocalizedPath(false);
}
- private void checkLocalizedPath(String visibility)
+ private void appendStringArray(StringBuilder buffer, String[] data) {
+ if (data != null && data.length != 0) {
+ buffer.append(data[0]);
+ for(int i=1; i < data.length; i++) {
+ buffer.append(',');
+ buffer.append(data[i]);
+ }
+ }
+ }
+
+ private void appendUriArray(StringBuilder buffer, URI[] data) {
+ if (data != null && data.length != 0) {
+ buffer.append(data[0]);
+ for(int i=1; i < data.length; i++) {
+ buffer.append(',');
+ buffer.append(data[i]);
+ }
+ }
+ }
+
+ private void dumpState(Configuration conf1) throws IOException {
+ StringBuilder buf = new StringBuilder();
+ buf.append("\nFiles:");
+ appendUriArray(buf, DistributedCache.getCacheFiles(conf1));
+ buf.append("\nArchives:");
+ appendUriArray(buf, DistributedCache.getCacheArchives(conf1));
+ buf.append("\nFile Visible:");
+ appendStringArray(buf, TrackerDistributedCacheManager.getFileVisibilities
+ (conf1));
+ buf.append("\nArchive Visible:");
+ appendStringArray(buf, TrackerDistributedCacheManager.getArchiveVisibilities
+ (conf1));
+ buf.append("\nFile timestamps:");
+ appendStringArray(buf, DistributedCache.getFileTimestamps(conf1));
+ buf.append("\nArchive timestamps:");
+ appendStringArray(buf, DistributedCache.getArchiveTimestamps(conf1));
+ LOG.info("state = " + buf.toString());
+ }
+
+ private void checkLocalizedPath(boolean visibility)
throws IOException, LoginException, InterruptedException {
TrackerDistributedCacheManager manager =
new TrackerDistributedCacheManager(conf, taskController);
String userName = getJobOwnerName();
File workDir = new File(TEST_ROOT_DIR, "workdir");
Path cacheFile = new Path(TEST_ROOT_DIR, "fourthcachefile");
- if ("true".equals(visibility)) {
+ if (visibility) {
createPublicTempFile(cacheFile);
} else {
createPrivateTempFile(cacheFile);
@@ -319,6 +358,7 @@ public class TestTrackerDistributedCache
DistributedCache.addCacheFile(cacheFile.toUri(), conf1);
TrackerDistributedCacheManager.determineTimestamps(conf1);
TrackerDistributedCacheManager.determineCacheVisibilities(conf1);
+ dumpState(conf1);
// Task localizing for job
TaskDistributedCacheManager handle = manager
@@ -328,7 +368,7 @@ public class TestTrackerDistributedCache
TaskTracker.getPublicDistributedCacheDir());
TaskDistributedCacheManager.CacheFile c = handle.getCacheFiles().get(0);
String distCacheDir;
- if ("true".equals(visibility)) {
+ if (visibility) {
distCacheDir = TaskTracker.getPublicDistributedCacheDir();
} else {
distCacheDir = TaskTracker.getPrivateDistributedCacheDir(userName);
@@ -337,19 +377,19 @@ public class TestTrackerDistributedCache
manager.getLocalCache(cacheFile.toUri(), conf1, distCacheDir,
fs.getFileStatus(cacheFile), false,
c.timestamp, new Path(TEST_ROOT_DIR), false,
- Boolean.parseBoolean(visibility));
+ visibility);
assertTrue("Cache file didn't get localized in the expected directory. " +
"Expected localization to happen within " +
ROOT_MAPRED_LOCAL_DIR + "/" + distCacheDir +
", but was localized at " +
localizedPath, localizedPath.toString().contains(distCacheDir));
- if ("true".equals(visibility)) {
+ if (visibility) {
checkPublicFilePermissions(new Path[]{localizedPath});
} else {
checkFilePermissions(new Path[]{localizedPath});
}
}
-
+
/**
* Check proper permissions on the cache files
*
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java?rev=1077420&r1=1077419&r2=1077420&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/fs/TestFileSystem.java
Fri Mar 4 04:13:04 2011
@@ -647,28 +647,26 @@ public class TestFileSystem extends Test
assertNotSame(fsA, fsB);
Token<T> t1 = mock(Token.class);
- ugiA = UserGroupInformation.createRemoteUser("foo");
- ugiA.addToken(t1);
+ UserGroupInformation ugiA2 = UserGroupInformation.createRemoteUser("foo");
- fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ fsA = ugiA2.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run() throws Exception {
return FileSystem.get(new URI("cachedfile://a"), conf);
}
});
- //Although the users in the UGI are same, ugiA has tokens in it, and
- //we should end up with different filesystems corresponding to the two UGIs
+ // Although the users in the UGI are same, they have different subjects
+ // and so are different.
assertNotSame(fsA, fsA1);
- ugiA = UserGroupInformation.createRemoteUser("foo");
ugiA.addToken(t1);
- fsA1 = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ fsA = ugiA.doAs(new PrivilegedExceptionAction<FileSystem>() {
public FileSystem run() throws Exception {
return FileSystem.get(new URI("cachedfile://a"), conf);
}
});
- //Now the users in the UGI are the same, and they also have the same token.
- //We should have the same filesystem for both
+ // Make sure that different UGI's with the same subject lead to the same
+ // file system.
assertSame(fsA, fsA1);
}
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java?rev=1077420&r1=1077419&r2=1077420&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestUserGroupInformation.java
Fri Mar 4 04:13:04 2011
@@ -156,10 +156,17 @@ public class TestUserGroupInformation {
UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES);
assertEquals(uugi, uugi);
- // The subjects should be equal, so this should work
- assertTrue(uugi.equals(
- UserGroupInformation.createUserForTesting
- (USER_NAME, GROUP_NAMES)));
+ // The subjects should be different, so this should fail
+ UserGroupInformation ugi2 =
+ UserGroupInformation.createUserForTesting(USER_NAME, GROUP_NAMES);
+ assertFalse(uugi.equals(ugi2));
+ assertFalse(uugi.hashCode() == ugi2.hashCode());
+
+ // two ugi that have the same subject need to be equal
+ UserGroupInformation ugi3 = new UserGroupInformation(uugi.getSubject());
+ assertEquals(uugi, ugi3);
+ assertEquals(uugi.hashCode(), ugi3.hashCode());
+
// ensure that different UGI with the same subject are equal
assertEquals(uugi, new UserGroupInformation(uugi.getSubject()));
}
@@ -172,8 +179,8 @@ public class TestUserGroupInformation {
"RealUser", GROUP_NAMES);
UserGroupInformation proxyUgi1 = UserGroupInformation.createProxyUser(
USER_NAME, realUgi1);
- UserGroupInformation proxyUgi2 = UserGroupInformation.createProxyUser(
- USER_NAME, realUgi2);
+ UserGroupInformation proxyUgi2 =
+ new UserGroupInformation( proxyUgi1.getSubject());
UserGroupInformation remoteUgi = UserGroupInformation.createRemoteUser(USER_NAME);
assertEquals(proxyUgi1, proxyUgi2);
assertFalse(remoteUgi.equals(proxyUgi1));
@@ -284,16 +291,16 @@ public class TestUserGroupInformation {
return null;
}
});
- UserGroupInformation proxyUgi2 = UserGroupInformation.createProxyUser(
- "proxy", ugi);
+ UserGroupInformation proxyUgi2 =
+ new UserGroupInformation(proxyUgi.getSubject());
proxyUgi2.setAuthenticationMethod(AuthenticationMethod.PROXY);
Assert.assertEquals(proxyUgi, proxyUgi2);
// Equality should work if authMethod is null
UserGroupInformation realugi = UserGroupInformation.getCurrentUser();
UserGroupInformation proxyUgi3 = UserGroupInformation.createProxyUser(
"proxyAnother", realugi);
- UserGroupInformation proxyUgi4 = UserGroupInformation.createProxyUser(
- "proxyAnother", realugi);
+ UserGroupInformation proxyUgi4 =
+ new UserGroupInformation(proxyUgi3.getSubject());
Assert.assertEquals(proxyUgi3, proxyUgi4);
}
}
|