manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1558893 - in /manifoldcf/branches/release-1.5-branch: ./ connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/ connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/ connectors/rs...
Date Thu, 16 Jan 2014 19:25:50 GMT
Author: kwright
Date: Thu Jan 16 19:25:49 2014
New Revision: 1558893

URL: http://svn.apache.org/r1558893
Log:
Pull up part of fix for CONNECTORS-858 from trunk.

Modified:
    manifoldcf/branches/release-1.5-branch/   (props changed)
    manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
    manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java
    manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
    manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
    manifoldcf/branches/release-1.5-branch/connectors/rss/   (props changed)
    manifoldcf/branches/release-1.5-branch/connectors/sharepoint/   (props changed)
    manifoldcf/branches/release-1.5-branch/connectors/wiki/   (props changed)
    manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/Javascript.py
  (props changed)
    manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py
  (props changed)

Propchange: manifoldcf/branches/release-1.5-branch/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk:r1558892

Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
(original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
Thu Jan 16 19:25:49 2014
@@ -33,7 +33,6 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -189,26 +188,13 @@ public class HDFSOutputConnector extends
       if (user == null)
         throw new ManifoldCFException("User must be specified");
       
-      String nameNode = "hdfs://"+nameNodeHost+":"+nameNodePort;
+      String nameNode = "file://"+nameNodeHost+":"+nameNodePort;
       //System.out.println("Namenode = '"+nameNode+"'");
 
       /*
-       * make Configuration
-       */
-      Configuration config = null;
-      ClassLoader ocl = Thread.currentThread().getContextClassLoader();
-      try {
-        Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
-        config = new Configuration();
-        config.set("fs.default.name", nameNode);
-      } finally {
-        Thread.currentThread().setContextClassLoader(ocl);
-      }
-      
-      /*
        * get connection to HDFS
        */
-      GetSessionThread t = new GetSessionThread(nameNode,config,user);
+      GetSessionThread t = new GetSessionThread(nameNode,user);
       try {
         t.start();
         t.finishUp();
@@ -770,15 +756,13 @@ public class HDFSOutputConnector extends
 
   protected static class GetSessionThread extends Thread {
     protected final String nameNode;
-    protected final Configuration config;
     protected final String user;
     protected Throwable exception = null;
     protected HDFSSession session = null;
 
-    public GetSessionThread(String nameNode, Configuration config, String user) {
+    public GetSessionThread(String nameNode, String user) {
       super();
       this.nameNode = nameNode;
-      this.config = config;
       this.user = user;
       setDaemon(true);
     }
@@ -786,7 +770,7 @@ public class HDFSOutputConnector extends
     public void run() {
       try {
         // Create a session
-        session = new HDFSSession(nameNode, config, user);
+        session = new HDFSSession(nameNode, user);
       } catch (Throwable e) {
         this.exception = e;
       }

Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java
(original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java
Thu Jan 16 19:25:49 2014
@@ -42,16 +42,24 @@ import java.util.HashMap;
  */
 public class HDFSSession {
 
-  private FileSystem fileSystem;
+  private final FileSystem fileSystem;
   private final String nameNode;
   private final Configuration config;
   private final String user;
   
-  public HDFSSession(String nameNode, Configuration config, String user) throws URISyntaxException,
IOException, InterruptedException {
+  public HDFSSession(String nameNode, String user) throws URISyntaxException, IOException,
InterruptedException {
     this.nameNode = nameNode;
-    this.config = config;
     this.user = user;
-    fileSystem = FileSystem.get(new URI(nameNode), config, user);
+    // Switch class loaders so that scheme registration works properly
+    ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+    try {
+      Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
+      config = new Configuration();
+      config.set("fs.defaultFS", nameNode);
+      fileSystem = FileSystem.get(new URI(nameNode), config, user);
+    } finally {
+      Thread.currentThread().setContextClassLoader(ocl);
+    }
   }
 
   public Map<String, String> getRepositoryInfo() {
@@ -61,10 +69,10 @@ public class HDFSSession {
     info.put("Config", config.toString());
     info.put("User", user);
     info.put("Canonical Service Name", fileSystem.getCanonicalServiceName());
-    info.put("Default Block Size", Long.toString(fileSystem.getDefaultBlockSize()));
-    info.put("Default Replication", Short.toString(fileSystem.getDefaultReplication()));
-    info.put("Home Directory", fileSystem.getHomeDirectory().toUri().toString());
-    info.put("Working Directory", fileSystem.getWorkingDirectory().toUri().toString());
+    //info.put("Default Block Size", Long.toString(fileSystem.getDefaultBlockSize()));
+    //info.put("Default Replication", Short.toString(fileSystem.getDefaultReplication()));
+    //info.put("Home Directory", fileSystem.getHomeDirectory().toUri().toString());
+    //info.put("Working Directory", fileSystem.getWorkingDirectory().toUri().toString());
     return info;
   }
 

Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
(original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
Thu Jan 16 19:25:49 2014
@@ -19,7 +19,6 @@
 package org.apache.manifoldcf.crawler.connectors.hdfs;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
 import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
@@ -164,22 +163,9 @@ public class HDFSRepositoryConnector ext
         Logging.connectors.debug("HDFS: User = '" + user + "'");
       }
 
-      String nameNode = "hdfs://"+nameNodeHost+":"+nameNodePort;
+      String nameNode = "file://"+nameNodeHost+":"+nameNodePort;
 
-      /*
-       * make Configuration
-       */
-      Configuration config = null;
-      ClassLoader ocl = Thread.currentThread().getContextClassLoader();
-      try {
-        Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
-        config = new Configuration();
-        config.set("fs.default.name", nameNode);
-      } finally {
-        Thread.currentThread().setContextClassLoader(ocl);
-      }
-      
-      GetSessionThread t = new GetSessionThread(nameNode,config,user);
+      GetSessionThread t = new GetSessionThread(nameNode,user);
       try {
         t.start();
         t.finishUp();
@@ -1692,15 +1678,13 @@ public class HDFSRepositoryConnector ext
 
   protected static class GetSessionThread extends Thread {
     protected final String nameNode;
-    protected final Configuration config;
     protected final String user;
     protected Throwable exception = null;
     protected HDFSSession session;
 
-    public GetSessionThread(String nameNode, Configuration config, String user) {
+    public GetSessionThread(String nameNode, String user) {
       super();
       this.nameNode = nameNode;
-      this.config = config;
       this.user = user;
       setDaemon(true);
     }
@@ -1708,7 +1692,7 @@ public class HDFSRepositoryConnector ext
     public void run() {
       try {
         // Create a session
-        session = new HDFSSession(nameNode, config, user);
+        session = new HDFSSession(nameNode, user);
       } catch (Throwable e) {
         this.exception = e;
       }

Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
(original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
Thu Jan 16 19:25:49 2014
@@ -43,18 +43,30 @@ import java.util.HashMap;
  */
 public class HDFSSession {
 
-  private FileSystem fileSystem;
-  private String nameNode;
-  private Configuration config;
-  private String user;
+  private final FileSystem fileSystem;
+  private final String nameNode;
+  private final Configuration config;
+  private final String user;
   
-  public HDFSSession(String nameNode, Configuration config, String user) throws URISyntaxException,
IOException, InterruptedException {
+  public HDFSSession(String nameNode, String user) throws URISyntaxException, IOException,
InterruptedException {
     this.nameNode = nameNode;
-    this.config = config;
     this.user = user;
-    fileSystem = FileSystem.get(new URI(nameNode), config, user);
+    // Switch class loaders so that scheme registration works properly
+    ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+    try {
+      Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
+      config = new Configuration();
+      config.set("fs.defaultFS", nameNode);
+      fileSystem = FileSystem.get(new URI(nameNode), config, user);
+    } finally {
+      Thread.currentThread().setContextClassLoader(ocl);
+    }
   }
 
+  public static void runMe()
+  {
+  }
+  
   public Map<String, String> getRepositoryInfo() {
     Map<String, String> info = new HashMap<String, String>();
 

Propchange: manifoldcf/branches/release-1.5-branch/connectors/rss/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/connectors/rss:r1558892

Propchange: manifoldcf/branches/release-1.5-branch/connectors/sharepoint/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/connectors/sharepoint:r1558892

Propchange: manifoldcf/branches/release-1.5-branch/connectors/wiki/
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/connectors/wiki:r1558892

Propchange: manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/Javascript.py
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/Javascript.py:r1558892

Propchange: manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py
------------------------------------------------------------------------------
  Merged /manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py:r1558892



Mime
View raw message