manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From min...@apache.org
Subject svn commit: r1495913 [1/5] - in /manifoldcf/branches/CONNECTORS-728: ./ connectors/ connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/ connectors/filesystem/connector/src/main/native2ascii/org/apache/mani...
Date Mon, 24 Jun 2013 03:26:58 GMT
Author: minoru
Date: Mon Jun 24 03:26:56 2013
New Revision: 1495913

URL: http://svn.apache.org/r1495913
Log:
Add HDFS Connector.

Added:
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputSpecs.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/Messages.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/Messages.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_en_US.properties   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/agents/output/hdfs/common_ja_JP.properties   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_en_US.properties   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/hdfs/common_ja_JP.properties   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.html   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editConfiguration.js   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editSpecification.html   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/editSpecification.js   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/viewConfiguration.html   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/resources/org/apache/manifoldcf/agents/output/hdfs/viewSpecification.html   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BaseDerby.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BaseHSQLDB.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BaseMySQL.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/BasePostgresql.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityDerbyTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityHSQLDBTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityMySQLTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/agents/output/hdfs/SanityPostgresqlTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BaseDerby.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BaseHSQLDB.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BaseMySQL.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/BasePostgresql.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityDerbyTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityHSQLDBTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityMySQLTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/test/java/org/apache/manifoldcf/crawler/connectors/hdfs/tests/SanityPostgresqlTest.java   (with props)
    manifoldcf/branches/CONNECTORS-728/connectors/hdfs/pom.xml   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/build.xml   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/pom.xml   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityCombinedHSQLDBIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityDerbyIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityHSQLDBIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityMySQLIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityPostgresqlIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/APISanityTester.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseDerby.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseHSQLDB.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseMySQL.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BasePostgresql.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIDerby.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BaseUIHSQLDB.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlDerbyLT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlHSQLDBLT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlMySQLLT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlPostgresqlLT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/BigCrawlTester.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationDerbyIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationHSQLDBIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationMySQLIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationPostgresqlIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/ExpirationTester.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/FileHelper.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountDerbyIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountHSQLDBIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountMySQLIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountPostgresqlIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/HopcountTester.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationCombinedHSQLDBUI.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationDerbyUI.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/NavigationUITester.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityDerbyIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityHSQLDBIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityMySQLIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityPostgresqlIT.java   (with props)
    manifoldcf/branches/CONNECTORS-728/tests/hdfs/src/test/java/org/apache/manifoldcf/hdfs_tests/SanityTester.java   (with props)
Modified:
    manifoldcf/branches/CONNECTORS-728/build.xml
    manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
    manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties
    manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties
    manifoldcf/branches/CONNECTORS-728/connectors/pom.xml

Modified: manifoldcf/branches/CONNECTORS-728/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/build.xml?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/build.xml (original)
+++ manifoldcf/branches/CONNECTORS-728/build.xml Mon Jun 24 03:26:56 2013
@@ -66,6 +66,7 @@
         <ant dir="connectors/filenet" target="clean"/>
         <ant dir="connectors/filesystem" target="clean"/>
         <ant dir="connectors/gts" target="clean"/>
+        <ant dir="connectors/hdfs" target="clean"/>
         <ant dir="connectors/jcifs" target="clean"/>
         <ant dir="connectors/jdbc" target="clean"/>
         <ant dir="connectors/livelink" target="clean"/>
@@ -86,6 +87,7 @@
         <ant dir="tests/cmis" target="clean"/>
         <ant dir="tests/filesystem" target="clean"/>
         <ant dir="tests/gts" target="clean"/>
+        <ant dir="tests/hdfs" target="clean"/>
         <ant dir="tests/opensearchserver" target="clean"/>
         <ant dir="tests/rss" target="clean"/>
         <ant dir="tests/solr" target="clean"/>
@@ -118,6 +120,7 @@
         <ant dir="connectors/filenet" target="clean"/>
         <ant dir="connectors/filesystem" target="clean"/>
         <ant dir="connectors/gts" target="clean"/>
+        <ant dir="connectors/hdfs" target="clean"/>
         <ant dir="connectors/jcifs" target="clean"/>
         <ant dir="connectors/jdbc" target="clean"/>
         <ant dir="connectors/livelink" target="clean"/>
@@ -138,6 +141,7 @@
         <ant dir="tests/cmis" target="clean"/>
         <ant dir="tests/filesystem" target="clean"/>
         <ant dir="tests/gts" target="clean"/>
+        <ant dir="tests/hdfs" target="clean"/>
         <ant dir="tests/opensearchserver" target="clean"/>
         <ant dir="tests/rss" target="clean"/>
         <ant dir="tests/solr" target="clean"/>
@@ -521,6 +525,42 @@
         <ant dir="connectors/gts" target="run-tests-HSQLDB"/>
     </target>
 
+    <target name="setup-hdfs-connector" depends="build-framework" if="downloaded"/>
+
+    <target name="setup-hdfs-connector-tests" depends="build-tests-framework" if="downloaded"/>
+
+    <target name="build-hdfs-connector" depends="setup-hdfs-connector" if="downloaded">
+        <ant dir="connectors/hdfs" target="build"/>
+    </target>
+
+    <target name="doc-hdfs-connector" depends="setup-hdfs-connector" if="downloaded">
+      <ant dir="connectors/hdfs" target="doc"/>
+    </target>
+
+    <target name="build-tests-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+        <ant dir="connectors/hdfs" target="build-tests"/>
+    </target>
+
+    <target name="run-tests-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+        <ant dir="connectors/hdfs" target="run-tests"/>
+    </target>
+
+    <target name="run-tests-derby-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+        <ant dir="connectors/hdfs" target="run-tests-derby"/>
+    </target>
+
+    <target name="run-tests-postgresql-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+        <ant dir="connectors/hdfs" target="run-tests-postgresql"/>
+    </target>
+
+    <target name="run-tests-mysql-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+        <ant dir="connectors/hdfs" target="run-tests-mysql"/>
+    </target>
+
+    <target name="run-tests-HSQLDB-hdfs-connector" depends="setup-hdfs-connector,setup-hdfs-connector-tests" if="downloaded">
+        <ant dir="connectors/hdfs" target="run-tests-HSQLDB"/>
+    </target>
+
     <target name="setup-jcifs-connector" depends="build-framework" if="downloaded"/>
 
     <target name="setup-jcifs-connector-tests" depends="build-tests-framework" if="downloaded"/>
@@ -1668,6 +1708,48 @@
         </condition>
     </target>
 
+    <target name="calculate-hdfs-condition" depends="build-hdfs-connector">
+        <available file="connectors/hdfs/dist/lib" type="dir" property="hdfs.exists"/>
+        <condition property="hdfs.include">
+            <and>
+                <isset property="hdfs.exists"/>
+                <isset property="downloaded"/>
+            </and>
+        </condition>
+    </target>
+
+    <target name="calculate-hdfs-doc-condition" depends="doc-hdfs-connector">
+        <available file="connectors/hdfs/dist/doc" type="dir" property="hdfs-doc.exists"/>
+        <condition property="hdfs-doc.include">
+            <and>
+                <isset property="hdfs-doc.exists"/>
+                <isset property="downloaded"/>
+            </and>
+        </condition>
+    </target>
+
+    <target name="deliver-hdfs-connector" depends="calculate-hdfs-condition" if="hdfs.include">
+        <antcall target="general-connector-delivery">
+            <param name="connector-name" value="hdfs"/>
+        </antcall>
+        <antcall target="general-add-repository-connector">
+            <param name="connector-name" value="hdfs"/>
+            <param name="connector-label" value="HDFS"/>
+            <param name="connector-class" value="org.apache.manifoldcf.crawler.connectors.hdfs.HDFSRepositoryConnector"/>
+        </antcall>
+        <antcall target="general-add-output-connector">
+            <param name="connector-name" value="hdfs"/>
+            <param name="connector-label" value="HDFS"/>
+            <param name="connector-class" value="org.apache.manifoldcf.agents.output.hdfs.HDFSOutputConnector"/>
+        </antcall>
+    </target>
+
+    <target name="deliver-hdfs-connector-doc" depends="calculate-hdfs-doc-condition" if="hdfs-doc.include">
+        <antcall target="general-connector-doc-delivery">
+            <param name="connector-name" value="hdfs"/>
+        </antcall>
+    </target>
+
     <target name="calculate-jdbc-doc-condition" depends="doc-jdbc-connector">
         <available file="connectors/jdbc/dist/doc" type="dir" property="jdbc-doc.exists"/>
         <condition property="jdbc-doc.include">
@@ -2223,7 +2305,16 @@
       </condition>
     </target>
 
-    <target name="calculate-jcifs-tests-condition" depends="calculate-jcifs-condition,calculate-nulloutput-condition">
+    <target name="calculate-hdfs-tests-condition" depends="calculate-hdfs-condition,calculate-nulloutput-condition">
+      <condition property="hdfs-tests.include">
+        <and>
+            <isset property="hdfs.include"/>
+            <isset property="nulloutput.include"/>
+        </and>
+      </condition>
+    </target>
+
+	<target name="calculate-jcifs-tests-condition" depends="calculate-jcifs-condition,calculate-nulloutput-condition">
       <condition property="jcifs-tests.include">
         <and>
             <isset property="jcifs.include"/>
@@ -2358,6 +2449,18 @@
         <ant dir="tests/filesystem" target="run-load-derby"/>
     </target>
 
+    <target name="run-hdfs-tests-derby" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-derby"/>
+    </target>
+
+    <target name="run-hdfs-UI-tests-derby" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-UI-derby"/>
+    </target>
+
+    <target name="run-hdfs-loadtests-derby" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-load-derby"/>
+    </target>
+
     <target name="run-jcifs-UI-tests-derby" depends="build-tests-framework,build-tests-jcifs-connector,build-tests-nulloutput-connector,calculate-jcifs-tests-condition" if="jcifs-tests.include">
         <ant dir="tests/jcifs" target="run-UI-derby"/>
     </target>
@@ -2502,6 +2605,22 @@
         <ant dir="tests/filesystem" target="run-load-mysql"/>
     </target>
 
+    <target name="run-hdfs-tests-postgresql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-postgresql"/>
+    </target>
+
+    <target name="run-hdfs-tests-mysql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-mysql"/>
+    </target>
+
+    <target name="run-hdfs-loadtests-postgresql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-load-postgresql"/>
+    </target>
+
+    <target name="run-hdfs-loadtests-mysql" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-load-mysql"/>
+    </target>
+
     <target name="run-wiki-tests-postgresql" depends="build-tests-framework,build-tests-wiki-connector,build-tests-nulloutput-connector,calculate-wiki-tests-condition" if="wiki-tests.include">
         <ant dir="tests/wiki" target="run-postgresql"/>
     </target>
@@ -2566,6 +2685,18 @@
         <ant dir="tests/filesystem" target="run-load-HSQLDB"/>
     </target>
 
+    <target name="run-hdfs-tests-HSQLDB" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-HSQLDB"/>
+    </target>
+
+    <target name="run-hdfs-UI-tests-HSQLDB" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-UI-HSQLDB"/>
+    </target>
+
+    <target name="run-hdfs-loadtests-HSQLDB" depends="build-tests-framework,build-tests-hdfs-connector,build-tests-nulloutput-connector,calculate-hdfs-tests-condition" if="hdfs-tests.include">
+        <ant dir="tests/hdfs" target="run-load-HSQLDB"/>
+    </target>
+
     <target name="run-wiki-tests-HSQLDB" depends="build-tests-framework,build-tests-wiki-connector,build-tests-nulloutput-connector,calculate-wiki-tests-condition" if="wiki-tests.include">
         <ant dir="tests/wiki" target="run-HSQLDB"/>
     </target>
@@ -2614,45 +2745,45 @@
         <ant dir="tests/sharepoint" target="run-load-HSQLDB"/>
     </target>
 
-    <target name="run-tests-open-connectors" depends="run-tests-activedirectory-connector,run-tests-ldap-connector,run-tests-alfresco-connector,run-tests-cmis-connector,run-tests-filesystem-connector,run-tests-nullauthority-connector,run-tests-nulloutput-connector,run-tests-rss-connector,run-tests-solr-connector,run-tests-webcrawler-connector,run-tests-wiki-connector,run-tests-jdbc-connector"/>
+    <target name="run-tests-open-connectors" depends="run-tests-activedirectory-connector,run-tests-ldap-connector,run-tests-alfresco-connector,run-tests-cmis-connector,run-tests-filesystem-connector,run-tests-hdfs-connector,run-tests-nullauthority-connector,run-tests-nulloutput-connector,run-tests-rss-connector,run-tests-solr-connector,run-tests-webcrawler-connector,run-tests-wiki-connector,run-tests-jdbc-connector"/>
     <target name="run-tests-lgpl-connectors" depends="run-tests-jcifs-connector"/>
     <target name="run-tests-proprietary-connectors" depends="run-tests-documentum-connector,run-tests-filenet-connector,run-tests-livelink-connector,run-tests-memex-connector,run-tests-meridio-connector,run-tests-sharepoint-connector"/>
 
-    <target name="run-tests-derby-open-connectors" depends="run-tests-derby-activedirectory-connector,run-tests-derby-ldap-connector,run-tests-derby-alfresco-connector,run-tests-derby-cmis-connector,run-tests-derby-filesystem-connector,run-tests-derby-nullauthority-connector,run-tests-derby-nulloutput-connector,run-tests-derby-rss-connector,run-tests-derby-solr-connector,run-tests-derby-webcrawler-connector,run-tests-derby-wiki-connector,run-tests-derby-jdbc-connector"/>
+    <target name="run-tests-derby-open-connectors" depends="run-tests-derby-activedirectory-connector,run-tests-derby-ldap-connector,run-tests-derby-alfresco-connector,run-tests-derby-cmis-connector,run-tests-derby-filesystem-connector,run-tests-derby-hdfs-connector,run-tests-derby-nullauthority-connector,run-tests-derby-nulloutput-connector,run-tests-derby-rss-connector,run-tests-derby-solr-connector,run-tests-derby-webcrawler-connector,run-tests-derby-wiki-connector,run-tests-derby-jdbc-connector"/>
     <target name="run-tests-derby-lgpl-connectors" depends="run-tests-derby-jcifs-connector"/>
     <target name="run-tests-derby-proprietary-connectors" depends="run-tests-derby-documentum-connector,run-tests-derby-filenet-connector,run-tests-derby-livelink-connector,run-tests-derby-memex-connector,run-tests-derby-meridio-connector,run-tests-derby-sharepoint-connector"/>
     
-    <target name="end-to-end-tests-derby" depends="run-filesystem-tests-derby,run-webcrawler-tests-derby,run-rss-tests-derby,run-solr-tests-derby,run-wiki-tests-derby,run-alfresco-tests-derby,run-cmis-tests-derby,run-sharepoint-tests-derby"/>
+    <target name="end-to-end-tests-derby" depends="run-filesystem-tests-derby,run-hdfs-tests-derby,run-webcrawler-tests-derby,run-rss-tests-derby,run-solr-tests-derby,run-wiki-tests-derby,run-alfresco-tests-derby,run-cmis-tests-derby,run-sharepoint-tests-derby"/>
 
-    <target name="run-tests-postgresql-open-connectors" depends="run-tests-postgresql-activedirectory-connector,run-tests-postgresql-ldap-connector,run-tests-postgresql-alfresco-connector,run-tests-postgresql-cmis-connector,run-tests-postgresql-filesystem-connector,run-tests-postgresql-nullauthority-connector,run-tests-postgresql-nulloutput-connector,run-tests-postgresql-rss-connector,run-tests-postgresql-solr-connector,run-tests-postgresql-webcrawler-connector,run-tests-postgresql-wiki-connector,run-tests-postgresql-jdbc-connector,run-tests-postgresql-opensearchserver-connector,run-tests-postgresql-elasticsearch-connector"/>
+    <target name="run-tests-postgresql-open-connectors" depends="run-tests-postgresql-activedirectory-connector,run-tests-postgresql-ldap-connector,run-tests-postgresql-alfresco-connector,run-tests-postgresql-cmis-connector,run-tests-postgresql-filesystem-connector,run-tests-postgresql-hdfs-connector,run-tests-postgresql-nullauthority-connector,run-tests-postgresql-nulloutput-connector,run-tests-postgresql-rss-connector,run-tests-postgresql-solr-connector,run-tests-postgresql-webcrawler-connector,run-tests-postgresql-wiki-connector,run-tests-postgresql-jdbc-connector,run-tests-postgresql-opensearchserver-connector,run-tests-postgresql-elasticsearch-connector"/>
     <target name="run-tests-postgresql-lgpl-connectors" depends="run-tests-postgresql-jcifs-connector"/>
     <target name="run-tests-postgresql-proprietary-connectors" depends="run-tests-postgresql-documentum-connector,run-tests-postgresql-filenet-connector,run-tests-postgresql-livelink-connector,run-tests-postgresql-memex-connector,run-tests-postgresql-meridio-connector,run-tests-postgresql-sharepoint-connector"/>
     
-    <target name="end-to-end-tests-postgresql" depends="run-filesystem-tests-postgresql,run-webcrawler-tests-postgresql,run-wiki-tests-postgresql,run-alfresco-tests-postgresql,run-cmis-tests-postgresql,run-sharepoint-tests-postgresql"/>
+    <target name="end-to-end-tests-postgresql" depends="run-filesystem-tests-postgresql,run-hdfs-tests-postgresql,run-webcrawler-tests-postgresql,run-wiki-tests-postgresql,run-alfresco-tests-postgresql,run-cmis-tests-postgresql,run-sharepoint-tests-postgresql"/>
 
-    <target name="run-tests-mysql-open-connectors" depends="run-tests-mysql-activedirectory-connector,run-tests-mysql-ldap-connector,run-tests-mysql-alfresco-connector,run-tests-mysql-cmis-connector,run-tests-mysql-filesystem-connector,run-tests-mysql-nullauthority-connector,run-tests-mysql-nulloutput-connector,run-tests-mysql-rss-connector,run-tests-mysql-solr-connector,run-tests-mysql-webcrawler-connector,run-tests-mysql-wiki-connector,run-tests-mysql-jdbc-connector,run-tests-mysql-opensearchserver-connector,run-tests-mysql-elasticsearch-connector"/>
+    <target name="run-tests-mysql-open-connectors" depends="run-tests-mysql-activedirectory-connector,run-tests-mysql-ldap-connector,run-tests-mysql-alfresco-connector,run-tests-mysql-cmis-connector,run-tests-mysql-filesystem-connector,run-tests-mysql-hdfs-connector,run-tests-mysql-nullauthority-connector,run-tests-mysql-nulloutput-connector,run-tests-mysql-rss-connector,run-tests-mysql-solr-connector,run-tests-mysql-webcrawler-connector,run-tests-mysql-wiki-connector,run-tests-mysql-jdbc-connector,run-tests-mysql-opensearchserver-connector,run-tests-mysql-elasticsearch-connector"/>
     <target name="run-tests-mysql-lgpl-connectors" depends="run-tests-mysql-jcifs-connector"/>
     <target name="run-tests-mysql-proprietary-connectors" depends="run-tests-mysql-documentum-connector,run-tests-mysql-filenet-connector,run-tests-mysql-livelink-connector,run-tests-mysql-memex-connector,run-tests-mysql-meridio-connector,run-tests-mysql-sharepoint-connector"/>
     
-    <target name="end-to-end-tests-mysql" depends="run-filesystem-tests-mysql,run-webcrawler-tests-mysql,run-wiki-tests-mysql,run-alfresco-tests-mysql,run-cmis-tests-mysql,run-sharepoint-tests-mysql"/>
+    <target name="end-to-end-tests-mysql" depends="run-filesystem-tests-mysql,run-hdfs-tests-mysql,run-webcrawler-tests-mysql,run-wiki-tests-mysql,run-alfresco-tests-mysql,run-cmis-tests-mysql,run-sharepoint-tests-mysql"/>
 
-    <target name="run-tests-HSQLDB-open-connectors" depends="run-tests-HSQLDB-activedirectory-connector,run-tests-HSQLDB-ldap-connector,run-tests-HSQLDB-alfresco-connector,run-tests-HSQLDB-cmis-connector,run-tests-HSQLDB-filesystem-connector,run-tests-HSQLDB-nullauthority-connector,run-tests-HSQLDB-nulloutput-connector,run-tests-HSQLDB-rss-connector,run-tests-HSQLDB-solr-connector,run-tests-HSQLDB-webcrawler-connector,run-tests-HSQLDB-wiki-connector,run-tests-HSQLDB-jdbc-connector,run-tests-HSQLDB-opensearchserver-connector,run-tests-HSQLDB-elasticsearch-connector"/>
+    <target name="run-tests-HSQLDB-open-connectors" depends="run-tests-HSQLDB-activedirectory-connector,run-tests-HSQLDB-ldap-connector,run-tests-HSQLDB-alfresco-connector,run-tests-HSQLDB-cmis-connector,run-tests-HSQLDB-filesystem-connector,run-tests-HSQLDB-hdfs-connector,run-tests-HSQLDB-nullauthority-connector,run-tests-HSQLDB-nulloutput-connector,run-tests-HSQLDB-rss-connector,run-tests-HSQLDB-solr-connector,run-tests-HSQLDB-webcrawler-connector,run-tests-HSQLDB-wiki-connector,run-tests-HSQLDB-jdbc-connector,run-tests-HSQLDB-opensearchserver-connector,run-tests-HSQLDB-elasticsearch-connector"/>
     <target name="run-tests-HSQLDB-lgpl-connectors" depends="run-tests-HSQLDB-jcifs-connector"/>
     <target name="run-tests-HSQLDB-proprietary-connectors" depends="run-tests-HSQLDB-documentum-connector,run-tests-HSQLDB-filenet-connector,run-tests-HSQLDB-livelink-connector,run-tests-HSQLDB-memex-connector,run-tests-HSQLDB-meridio-connector,run-tests-HSQLDB-sharepoint-connector"/>
     
-    <target name="end-to-end-tests-HSQLDB" depends="run-filesystem-tests-HSQLDB,run-rss-tests-HSQLDB,run-wiki-tests-HSQLDB,run-alfresco-tests-HSQLDB,run-cmis-tests-HSQLDB,run-sharepoint-tests-HSQLDB"/>
+    <target name="end-to-end-tests-HSQLDB" depends="run-filesystem-tests-HSQLDB,run-hdfs-tests-HSQLDB,run-rss-tests-HSQLDB,run-wiki-tests-HSQLDB,run-alfresco-tests-HSQLDB,run-cmis-tests-HSQLDB,run-sharepoint-tests-HSQLDB"/>
 
-    <target name="end-to-end-loadtests-derby" depends="run-filesystem-loadtests-derby,run-rss-loadtests-derby,run-wiki-loadtests-derby,run-alfresco-loadtests-derby,run-cmis-loadtests-derby,run-sharepoint-loadtests-derby"/>
+    <target name="end-to-end-loadtests-derby" depends="run-filesystem-loadtests-derby,run-hdfs-loadtests-derby,run-rss-loadtests-derby,run-wiki-loadtests-derby,run-alfresco-loadtests-derby,run-cmis-loadtests-derby,run-sharepoint-loadtests-derby"/>
 
-    <target name="end-to-end-loadtests-postgresql" depends="run-filesystem-loadtests-postgresql,run-rss-loadtests-postgresql,run-wiki-loadtests-postgresql,run-alfresco-loadtests-postgresql,run-cmis-loadtests-postgresql,run-sharepoint-loadtests-postgresql"/>
+    <target name="end-to-end-loadtests-postgresql" depends="run-filesystem-loadtests-postgresql,run-hdfs-loadtests-postgresql,run-rss-loadtests-postgresql,run-wiki-loadtests-postgresql,run-alfresco-loadtests-postgresql,run-cmis-loadtests-postgresql,run-sharepoint-loadtests-postgresql"/>
 
-    <target name="end-to-end-loadtests-mysql" depends="run-filesystem-loadtests-mysql,run-rss-loadtests-mysql,run-wiki-loadtests-mysql,run-alfresco-loadtests-mysql,run-cmis-loadtests-mysql,run-sharepoint-loadtests-mysql"/>
+    <target name="end-to-end-loadtests-mysql" depends="run-filesystem-loadtests-mysql,run-hdfs-loadtests-mysql,run-rss-loadtests-mysql,run-wiki-loadtests-mysql,run-alfresco-loadtests-mysql,run-cmis-loadtests-mysql,run-sharepoint-loadtests-mysql"/>
 
-    <target name="end-to-end-loadtests-HSQLDB" depends="run-filesystem-loadtests-HSQLDB,run-rss-loadtests-HSQLDB,run-wiki-loadtests-HSQLDB,run-alfresco-loadtests-HSQLDB,run-cmis-loadtests-HSQLDB,run-sharepoint-loadtests-HSQLDB"/>
+    <target name="end-to-end-loadtests-HSQLDB" depends="run-filesystem-loadtests-HSQLDB,run-hdfs-loadtests-HSQLDB,run-rss-loadtests-HSQLDB,run-wiki-loadtests-HSQLDB,run-alfresco-loadtests-HSQLDB,run-cmis-loadtests-HSQLDB,run-sharepoint-loadtests-HSQLDB"/>
 
 
-    <target name="deliver-open-connectors" depends="deliver-googledrive-connector,deliver-dropbox-connector,deliver-nullauthority-connector,deliver-activedirectory-connector,deliver-ldap-connector,deliver-alfresco-connector,deliver-cmis-connector,deliver-filesystem-connector,deliver-rss-connector,deliver-webcrawler-connector,deliver-wiki-connector,deliver-jdbc-connector"/>
-    <target name="deliver-open-connectors-doc" depends="deliver-googledrive-connector-doc,deliver-dropbox-connector-doc,deliver-nullauthority-connector-doc,deliver-activedirectory-connector-doc,deliver-ldap-connector-doc,deliver-alfresco-connector-doc,deliver-cmis-connector-doc,deliver-filesystem-connector-doc,deliver-rss-connector-doc,deliver-webcrawler-connector-doc,deliver-wiki-connector-doc,deliver-jdbc-connector-doc"/>
+    <target name="deliver-open-connectors" depends="deliver-googledrive-connector,deliver-dropbox-connector,deliver-nullauthority-connector,deliver-activedirectory-connector,deliver-ldap-connector,deliver-alfresco-connector,deliver-cmis-connector,deliver-filesystem-connector,deliver-hdfs-connector,deliver-rss-connector,deliver-webcrawler-connector,deliver-wiki-connector,deliver-jdbc-connector"/>
+    <target name="deliver-open-connectors-doc" depends="deliver-googledrive-connector-doc,deliver-dropbox-connector-doc,deliver-nullauthority-connector-doc,deliver-activedirectory-connector-doc,deliver-ldap-connector-doc,deliver-alfresco-connector-doc,deliver-cmis-connector-doc,deliver-filesystem-connector-doc,deliver-hdfs-connector-doc,deliver-rss-connector-doc,deliver-webcrawler-connector-doc,deliver-wiki-connector-doc,deliver-jdbc-connector-doc"/>
     
     <target name="deliver-output-connectors" depends="deliver-gts-connector,deliver-solr-connector,deliver-nulloutput-connector,deliver-opensearchserver-connector,deliver-elasticsearch-connector"/>
     <target name="deliver-output-connectors-doc" depends="deliver-gts-connector-doc,deliver-solr-connector-doc,deliver-nulloutput-connector-doc,deliver-opensearchserver-connector-doc,deliver-elasticsearch-connector-doc"/>
@@ -2803,7 +2934,7 @@
 
     <target name="ldtest" depends="load-dr,load-hs"/>
 
-    <target name="uitest" depends="run-filesystem-UI-tests-derby,run-filesystem-UI-tests-HSQLDB,run-jcifs-UI-tests-derby,run-jdbc-UI-tests-derby,run-activedirectory-UI-tests-derby,run-ldap-UI-tests-derby,run-rss-UI-tests-derby,run-webcrawler-UI-tests-derby,run-wiki-UI-tests-derby,run-solr-UI-tests-derby,run-cmis-UI-tests-derby,run-gts-UI-tests-derby,run-opensearchserver-UI-tests-derby"/>
+    <target name="uitest" depends="run-filesystem-UI-tests-derby,run-hdfs-UI-tests-derby,run-filesystem-UI-tests-HSQLDB,run-hdfs-UI-tests-HSQLDB,run-jcifs-UI-tests-derby,run-jdbc-UI-tests-derby,run-activedirectory-UI-tests-derby,run-ldap-UI-tests-derby,run-rss-UI-tests-derby,run-webcrawler-UI-tests-derby,run-wiki-UI-tests-derby,run-solr-UI-tests-derby,run-cmis-UI-tests-derby,run-gts-UI-tests-derby,run-opensearchserver-UI-tests-derby"/>
     
     <target name="all" depends="build,javadoc,doc,image,test-dr,test-hs"/>
 
@@ -3200,6 +3331,12 @@ Use Apache Forrest version forrest-0.9-d
             <param name="artifact-name" value="commons-logging"/>
             <param name="artifact-type" value="jar"/>
         </antcall>
+        <antcall target="download-via-maven"><param name="target" value="lib"/>
+            <param name="project-path" value="commons-configuration"/>
+            <param name="artifact-version" value="1.6"/>
+            <param name="artifact-name" value="commons-configuration"/>
+            <param name="artifact-type" value="jar"/>
+        </antcall>
     </target>
   
     <target name="download-slf4j">
@@ -3743,7 +3880,18 @@ Use Apache Forrest version forrest-0.9-d
         </copy>
     </target>
 
-    <target name="download-solrj">
+    <target name="download-hadoop">
+        <mkdir dir="lib"/>
+        <antcall target="download-via-maven">
+            <param name="target" value="lib"/>
+            <param name="project-path" value="org/apache/hadoop"/>
+            <param name="artifact-version" value="1.0.4"/>
+            <param name="artifact-name" value="hadoop-core"/>
+            <param name="artifact-type" value="jar"/>
+        </antcall>
+    </target>
+    
+	  <target name="download-solrj">
         <mkdir dir="lib"/>
         <antcall target="download-via-maven">
             <param name="target" value="lib"/>
@@ -3761,7 +3909,7 @@ Use Apache Forrest version forrest-0.9-d
         </antcall>
     </target>
 
-    <target name="make-core-deps" depends="download-google-api-client,download-dropbox-client,download-solrj,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj">
+    <target name="make-core-deps" depends="download-google-api-client,download-dropbox-client,download-solrj,download-httpcomponents,download-json,download-hsqldb,download-xerces,download-commons,download-elasticsearch-plugin,download-solr-plugins,download-sharepoint-plugins,download-jstl,download-xmlgraphics-commons,download-wstx-asl,download-xmlsec,download-xml-apis,download-wss4j,download-velocity,download-streambuffer,download-stax,download-servlet-api,download-xml-resolver,download-osgi,download-opensaml,download-mimepull,download-mail,download-log4j,download-junit,download-jaxws,download-glassfish,download-jaxb,download-tomcat,download-h2,download-h2-support,download-geronimo-specs,download-fop,download-derby,download-postgresql,download-axis,download-saaj,download-wsdl4j,download-castor,download-jetty,download-slf4j,download-xalan,download-activation,download-avalon-framework,download-poi,download-chemistry,download-ecj,download-hadoop">
         <copy todir="lib">
             <fileset dir="lib-license" includes="*.txt"/>
         </copy>
@@ -3798,6 +3946,7 @@ Use Apache Forrest version forrest-0.9-d
         <ant dir="connectors/filenet" target="download-dependencies"/>
         <ant dir="connectors/filesystem" target="download-dependencies"/>
         <ant dir="connectors/gts" target="download-dependencies"/>
+        <ant dir="connectors/hdfs" target="download-dependencies"/>
         <ant dir="connectors/jcifs" target="download-dependencies"/>
         <ant dir="connectors/jdbc" target="download-dependencies"/>
         <ant dir="connectors/livelink" target="download-dependencies"/>
@@ -3836,6 +3985,7 @@ Use Apache Forrest version forrest-0.9-d
         <ant dir="connectors/filenet" target="download-cleanup"/>
         <ant dir="connectors/filesystem" target="download-cleanup"/>
         <ant dir="connectors/gts" target="download-cleanup"/>
+        <ant dir="connectors/hdfs" target="download-cleanup"/>
         <ant dir="connectors/jcifs" target="download-cleanup"/>
         <ant dir="connectors/jdbc" target="download-cleanup"/>
         <ant dir="connectors/livelink" target="download-cleanup"/>

Modified: manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/filesystem/FileConnector.java Mon Jun 24 03:26:56 2013
@@ -821,7 +821,7 @@ public class FileConnector extends org.a
 "  <tr>\n"+
 "    <td class=\"description\"><nobr>" + Messages.getBodyString(locale,"FileConnector.FilePathToURI") + "</nobr></td>\n"+
 "    <td class=\"value\">\n"+
-"      <input name=\"filepathtouri\" type=\"checkbox\" value=\"true\"" + (filePathToUri ? "checked" : "") +"/>\n" +
+"      <input name=\"filepathtouri\" type=\"checkbox\" value=\"true\"" + (filePathToUri ? "checked" : "") +"/>&nbsp;" + Messages.getBodyString(locale,"FileConnector.FilePathToURIExample") + "\n" +
 "    </td>\n"+
 "  </tr>\n"+
 "</table>\n"

Modified: manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_en_US.properties Mon Jun 24 03:26:56 2013
@@ -38,3 +38,4 @@ FileConnector.AddNewPath=Add new path
 
 FileConnector.FilePathToURITab=Convert file path to URI
 FileConnector.FilePathToURI=Convert file path to URI:
+FileConnector.FilePathToURIExample=EX) $REPOSITORY_PATH/http/localhost/index.html => http://localhost/index.html

Modified: manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties?rev=1495913&r1=1495912&r2=1495913&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties (original)
+++ manifoldcf/branches/CONNECTORS-728/connectors/filesystem/connector/src/main/native2ascii/org/apache/manifoldcf/crawler/connectors/filesystem/common_ja_JP.properties Mon Jun 24 03:26:56 2013
@@ -38,3 +38,4 @@ FileConnector.AddNewPath=新しã�
 
 FileConnector.FilePathToURITab=ファイルパスをURIへ変換する
 FileConnector.FilePathToURI=ファイルパスをURIへ変換する:
+FileConnector.FilePathToURIExample=例) $REPOSITORY_PATH/http/localhost/index.html => http://localhost/index.html

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml Mon Jun 24 03:26:56 2013
@@ -0,0 +1,39 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project name="hdfs" default="all">
+
+    <import file="../connector-build.xml"/>
+
+    <path id="connector-classpath">
+        <path refid="mcf-connector-build.connector-classpath"/>
+        <fileset dir="../../lib">
+            <include name="commons-configuration*.jar"/>
+            <include name="hadoop-core*.jar"/>
+        </fileset>
+    </path>
+
+    <target name="lib" depends="mcf-connector-build.lib,precompile-check" if="canBuild">
+        <mkdir dir="dist/lib"/>
+        <copy todir="dist/lib">
+            <fileset dir="../../lib">
+                <include name="commons-configuration*.jar"/>
+                <include name="hadoop-core*.jar"/>
+            </fileset>
+        </copy>
+    </target>
+</project>

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/build.xml
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,71 @@
+/* $Id: FileOutputConfig.java 1299512 2013-05-31 22:59:38Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+
+
+public class HDFSOutputConfig extends HDFSOutputParam {
+
+  /**
+   * 
+   */
+  private static final long serialVersionUID = -2062295503498352538L;
+
+  /** Parameters used for the configuration */
+  final private static ParameterEnum[] CONFIGURATIONLIST = {
+	  ParameterEnum.NAMENODE,
+	  ParameterEnum.USER
+  };
+
+  /** Build a set of ElasticSearchParameters by reading ConfigParams. If the
+   * value returned by ConfigParams.getParameter is null, the default value is
+   * set.
+   * 
+   * @param paramList
+   * @param params
+   */
+  public HDFSOutputConfig(ConfigParams params)
+  {
+    super(CONFIGURATIONLIST);
+    for (ParameterEnum param : CONFIGURATIONLIST) {
+      String value = params.getParameter(param.name());
+      if (value == null) {
+        value = param.defaultValue;
+      }
+      put(param, value);
+    }
+  }
+
+  /**
+   * @param variableContext
+   * @param parameters
+   */
+  public final static void contextToConfig(IPostParameters variableContext, ConfigParams parameters) {
+    for (ParameterEnum param : CONFIGURATIONLIST) {
+      String p = variableContext.getParameter(param.name().toLowerCase());
+      if (p != null) {
+        parameters.setParameter(param.name(), p);
+      }
+    }
+  }
+
+}

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConfig.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,554 @@
+/* $Id: FileOutputConnector.java 991374 2013-05-31 23:04:08Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URLEncoder;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.manifoldcf.agents.interfaces.IOutputAddActivity;
+import org.apache.manifoldcf.agents.interfaces.IOutputRemoveActivity;
+import org.apache.manifoldcf.agents.interfaces.OutputSpecification;
+import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
+import org.apache.manifoldcf.agents.interfaces.ServiceInterruption;
+import org.apache.manifoldcf.agents.output.BaseOutputConnector;
+import org.apache.manifoldcf.agents.output.hdfs.HDFSOutputParam.ParameterEnum;
+import org.apache.manifoldcf.core.interfaces.ConfigParams;
+import org.apache.manifoldcf.core.interfaces.ConfigurationNode;
+import org.apache.manifoldcf.core.interfaces.IHTTPOutput;
+import org.apache.manifoldcf.core.interfaces.IPostParameters;
+import org.apache.manifoldcf.core.interfaces.IThreadContext;
+import org.apache.manifoldcf.core.interfaces.ManifoldCFException;
+import org.apache.manifoldcf.core.interfaces.SpecificationNode;
+import org.json.JSONException;
+
+public class HDFSOutputConnector extends BaseOutputConnector {
+
+  public static final String _rcsid = "@(#)$Id: FileOutputConnector.java 988245 2010-08-23 18:39:35Z minoru $";
+
+  // Activities we log
+
+  /** Ingestion activity */
+  public final static String INGEST_ACTIVITY = "document ingest";
+  /** Document removal activity */
+  public final static String REMOVE_ACTIVITY = "document deletion";
+
+  // Activities list
+  protected static final String[] activitiesList = new String[]{INGEST_ACTIVITY, REMOVE_ACTIVITY};
+
+  /** Forward to the javascript to check the configuration parameters */
+  private static final String EDIT_CONFIGURATION_JS = "editConfiguration.js";
+
+  /** Forward to the HTML template to edit the configuration parameters */
+  private static final String EDIT_CONFIGURATION_HTML = "editConfiguration.html";
+
+  /** Forward to the HTML template to view the configuration parameters */
+  private static final String VIEW_CONFIGURATION_HTML = "viewConfiguration.html";
+
+  /** Forward to the javascript to check the specification parameters for the job */
+  private static final String EDIT_SPECIFICATION_JS = "editSpecification.js";
+
+  /** Forward to the template to edit the configuration parameters for the job */
+  private static final String EDIT_SPECIFICATION_HTML = "editSpecification.html";
+
+  /** Forward to the template to view the specification parameters for the job */
+  private static final String VIEW_SPECIFICATION_HTML = "viewSpecification.html";
+
+  protected Configuration config = null;
+  protected FileSystem fileSystem = null;
+
+  /** Constructor.
+   */
+  public HDFSOutputConnector() {
+  }
+
+  /** Return the list of activities that this connector supports (i.e. writes into the log).
+   *@return the list.
+   */
+  @Override
+  public String[] getActivitiesList() {
+    return activitiesList;
+  }
+
+  /** Connect.
+   *@param configParameters is the set of configuration parameters, which
+   * in this case describe the target appliance, basic auth configuration, etc.  (This formerly came
+   * out of the ini file.)
+   */
+  @Override
+  public void connect(ConfigParams configParams) {
+    super.connect(configParams);
+    
+    String nameNode = configParams.getParameter(ParameterEnum.NAMENODE.name());
+    
+    String user = configParams.getParameter(ParameterEnum.USER.name());
+    
+    /*
+     * make Configuration
+     */
+    ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+    try {
+      Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
+      config = new Configuration();
+      config.set("fs.default.name", nameNode);
+    } finally {
+      Thread.currentThread().setContextClassLoader(ocl);
+    }
+    
+    /*
+     * get connection to HDFS
+     */
+    try {
+        fileSystem = FileSystem.get(new URI(nameNode), config, user);
+  	} catch (URISyntaxException e) {
+        e.printStackTrace();
+  	} catch (IOException e) {
+        e.printStackTrace();
+  	} catch (InterruptedException e) {
+        e.printStackTrace();
+  	}
+  }
+
+  /** Close the connection.  Call this before discarding the connection.
+   */
+  @Override
+  public void disconnect() throws ManifoldCFException {
+    try {
+      fileSystem.close();
+    } catch(IOException ex) {
+      throw new ManifoldCFException(ex);
+    }
+    config.clear();
+    super.disconnect();
+  }
+
+  /** Set up a session */
+  protected void getSession() throws ManifoldCFException, ServiceInterruption {
+  }
+
+  /** Test the connection.  Returns a string describing the connection integrity.
+   *@return the connection's status as a displayable string.
+   */
+  @Override
+  public String check() throws ManifoldCFException {
+    try {
+      getSession();
+      return super.check();
+    } catch (ServiceInterruption e) {
+      return "Transient error: "+e.getMessage();
+    }
+  }
+
+  /** Get an output version string, given an output specification.  The output version string is used to uniquely describe the pertinent details of
+   * the output specification and the configuration, to allow the Connector Framework to determine whether a document will need to be output again.
+   * Note that the contents of the document cannot be considered by this method, and that a different version string (defined in IRepositoryConnector)
+   * is used to describe the version of the actual document.
+   *
+   * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be
+   * necessary.
+   *@param spec is the current output specification for the job that is doing the crawling.
+   *@return a string, of unlimited length, which uniquely describes output configuration and specification in such a way that if two such strings are equal,
+   * the document will not need to be sent again to the output data store.
+   */
+  @Override
+  public String getOutputDescription(OutputSpecification spec) throws ManifoldCFException, ServiceInterruption {
+    HDFSOutputSpecs specs = new HDFSOutputSpecs(getSpecNode(spec));
+    return specs.toJson().toString();
+  }
+
+  /** Add (or replace) a document in the output data store using the connector.
+   * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be
+   * necessary.
+   * The OutputSpecification is *not* provided to this method, because the goal is consistency, and if output is done it must be consistent with the
+   * output description, since that was what was partly used to determine if output should be taking place.  So it may be necessary for this method to decode
+   * an output description string in order to determine what should be done.
+   *@param documentURI is the URI of the document.  The URI is presumed to be the unique identifier which the output data store will use to process
+   * and serve the document.  This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.
+   *@param outputDescription is the description string that was constructed for this document by the getOutputDescription() method.
+   *@param document is the document data to be processed (handed to the output data store).
+   *@param authorityNameString is the name of the authority responsible for authorizing any access tokens passed in with the repository document.  May be null.
+   *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.
+   *@return the document status (accepted or permanently rejected).
+   */
+  @Override
+  public int addOrReplaceDocument(String documentURI, String outputDescription, RepositoryDocument document, String authorityNameString, IOutputAddActivity activities) throws ManifoldCFException, ServiceInterruption {
+    // Establish a session
+    getSession();
+
+    HDFSOutputConfig config = getConfigParameters(null);
+
+    HDFSOutputSpecs specs = null;
+    InputStream input = null;
+    FSDataOutputStream output = null;
+    FileLock lock = null;
+    try {
+      specs = new HDFSOutputSpecs(outputDescription);
+
+      /*
+       * make file path
+       */
+      StringBuffer strBuff = new StringBuffer();
+      if (specs.getRootPath() != null) {
+        strBuff.append(specs.getRootPath());
+      }
+      strBuff.append("/");
+      strBuff.append(documentURItoFilePath(documentURI));
+      Path path = new Path(strBuff.toString());
+
+      /*
+       * make directory
+       */
+      if (!fileSystem.exists(path.getParent())) {
+        fileSystem.mkdirs(path.getParent());
+      }
+
+      /*
+       * delete old file
+       */
+      if (fileSystem.exists(path)) {
+        fileSystem.delete(path, true);
+      }
+
+      input = document.getBinaryStream();
+      output = fileSystem.create(path);
+
+      /*
+       * write file
+       */
+      byte buf[] = new byte[1024];
+      int len;
+      while((len = input.read(buf)) != -1) {
+        output.write(buf, 0, len);
+      }
+      output.flush();
+    } catch (JSONException e) {
+      return DOCUMENTSTATUS_REJECTED;
+    } catch (URISyntaxException e) {
+      return DOCUMENTSTATUS_REJECTED;
+    } catch (NullPointerException e) {
+        return DOCUMENTSTATUS_REJECTED;
+    } catch (IOException e) {
+      return DOCUMENTSTATUS_REJECTED;
+    } finally {
+      try {
+        input.close();
+      } catch (IOException e) {
+      }
+      try {
+        output.close();
+      } catch (IOException e) {
+      }
+    }
+
+    activities.recordActivity(null, INGEST_ACTIVITY, new Long(document.getBinaryLength()), documentURI, "OK", null);
+    return DOCUMENTSTATUS_ACCEPTED;
+  }
+
+  /** Remove a document using the connector.
+   * Note that the last outputDescription is included, since it may be necessary for the connector to use such information to know how to properly remove the document.
+   *@param documentURI is the URI of the document.  The URI is presumed to be the unique identifier which the output data store will use to process
+   * and serve the document.  This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.
+   *@param outputDescription is the last description string that was constructed for this document by the getOutputDescription() method above.
+   *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.
+   */
+  @Override
+  public void removeDocument(String documentURI, String outputDescription, IOutputRemoveActivity activities) throws ManifoldCFException, ServiceInterruption {
+    // Establish a session
+    getSession();
+
+    HDFSOutputConfig config = getConfigParameters(null);
+
+    HDFSOutputSpecs specs = null;
+    try {
+      specs = new HDFSOutputSpecs(outputDescription);
+
+      /*
+       * make path
+       */
+      StringBuffer strBuff = new StringBuffer();
+      if (specs.getRootPath() != null) {
+        strBuff.append(specs.getRootPath());
+      }
+      strBuff.append("/");
+      strBuff.append(documentURItoFilePath(documentURI));
+      Path path = new Path(strBuff.toString());
+
+      /*
+       * delete old file
+       */
+      if (fileSystem.exists(path)) {
+        fileSystem.delete(path, true);
+      }
+    } catch (JSONException e) {
+    } catch (URISyntaxException e) {
+    } catch (NullPointerException e) {
+    } catch (IOException e) {
+    }
+
+    activities.recordActivity(null, REMOVE_ACTIVITY, null, documentURI, "OK", null);
+  }
+
+  /** Output the configuration header section.
+   * This method is called in the head section of the connector's configuration page.  Its purpose is to add the required tabs to the list, and to output any
+   * javascript methods that might be needed by the configuration editing HTML.
+   *@param threadContext is the local thread context.
+   *@param out is the output to which any HTML should be sent.
+   *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+   *@param tabsArray is an array of tab names.  Add to this array any tab names that are specific to the connector.
+   */
+  @Override
+  public void outputConfigurationHeader(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, List<String> tabsArray) throws ManifoldCFException, IOException {
+    super.outputConfigurationHeader(threadContext, out, locale, parameters, tabsArray);
+    tabsArray.add(Messages.getString(locale,"HDFSOutputConnector.ServerTabName"));
+    outputResource(EDIT_CONFIGURATION_JS, out, locale, null, null);
+  }
+
+  /** Output the configuration body section.
+   * This method is called in the body section of the connector's configuration page.  Its purpose is to present the required form elements for editing.
+   * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags.  The name of the
+   * form is "editconnection".
+   *@param threadContext is the local thread context.
+   *@param out is the output to which any HTML should be sent.
+   *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+   *@param tabName is the current tab name.
+   */
+  @Override
+  public void outputConfigurationBody(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters, String tabName) throws ManifoldCFException, IOException {
+    super.outputConfigurationBody(threadContext, out, locale, parameters, tabName);
+    HDFSOutputConfig config = this.getConfigParameters(parameters);
+    outputResource(EDIT_CONFIGURATION_HTML, out, locale, config, tabName);
+  }
+
+  /** Process a configuration post.
+   * This method is called at the start of the connector's configuration page, whenever there is a possibility that form data for a connection has been
+   * posted.  Its purpose is to gather form information and modify the configuration parameters accordingly.
+   * The name of the posted form is "editconnection".
+   *@param threadContext is the local thread context.
+   *@param variableContext is the set of variables available from the post, including binary file post information.
+   *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+   *@return null if all is well, or a string error message if there is an error that should prevent saving of the connection (and cause a redirection to an error page).
+   */
+  @Override
+  public String processConfigurationPost(IThreadContext threadContext, IPostParameters variableContext, Locale locale, ConfigParams parameters) throws ManifoldCFException {
+    HDFSOutputConfig.contextToConfig(variableContext, parameters);
+    return null;
+  }
+
+  /** View configuration.
+   * This method is called in the body section of the connector's view configuration page.  Its purpose is to present the connection information to the user.
+   * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+   *@param threadContext is the local thread context.
+   *@param out is the output to which any HTML should be sent.
+   *@param parameters are the configuration parameters, as they currently exist, for this connection being configured.
+   */
+  @Override
+  public void viewConfiguration(IThreadContext threadContext, IHTTPOutput out, Locale locale, ConfigParams parameters) throws ManifoldCFException, IOException {
+    outputResource(VIEW_CONFIGURATION_HTML, out, locale, getConfigParameters(parameters), null);
+  }
+
+  /** Output the specification header section.
+   * This method is called in the head section of a job page which has selected an output connection of the current type.  Its purpose is to add the required tabs
+   * to the list, and to output any javascript methods that might be needed by the job editing HTML.
+   *@param out is the output to which any HTML should be sent.
+   *@param os is the current output specification for this job.
+   *@param tabsArray is an array of tab names.  Add to this array any tab names that are specific to the connector.
+   */
+  @Override
+  public void outputSpecificationHeader(IHTTPOutput out, Locale locale, OutputSpecification os, List<String> tabsArray) throws ManifoldCFException, IOException {
+    super.outputSpecificationHeader(out, locale, os, tabsArray);
+    tabsArray.add(Messages.getString(locale, "HDFSOutputConnector.PathTabName"));
+    outputResource(EDIT_SPECIFICATION_JS, out, locale, null, null);
+  }
+
+  /** Output the specification body section.
+   * This method is called in the body section of a job page which has selected an output connection of the current type.  Its purpose is to present the required form elements for editing.
+   * The coder can presume that the HTML that is output from this configuration will be within appropriate <html>, <body>, and <form> tags.  The name of the
+   * form is "editjob".
+   *@param out is the output to which any HTML should be sent.
+   *@param os is the current output specification for this job.
+   *@param tabName is the current tab name.
+   */
+  @Override
+  public void outputSpecificationBody(IHTTPOutput out, Locale locale, OutputSpecification os, String tabName) throws ManifoldCFException, IOException {
+    super.outputSpecificationBody(out, locale, os, tabName);
+    HDFSOutputSpecs specs = getSpecParameters(os);
+    outputResource(EDIT_SPECIFICATION_HTML, out, locale, specs, tabName);
+  }
+
+  /** Process a specification post.
+   * This method is called at the start of job's edit or view page, whenever there is a possibility that form data for a connection has been
+   * posted.  Its purpose is to gather form information and modify the output specification accordingly.
+   * The name of the posted form is "editjob".
+   *@param variableContext contains the post data, including binary file-upload information.
+   *@param os is the current output specification for this job.
+   *@return null if all is well, or a string error message if there is an error that should prevent saving of the job (and cause a redirection to an error page).
+   */
+  @Override
+  public String processSpecificationPost(IPostParameters variableContext, Locale locale, OutputSpecification os) throws ManifoldCFException {
+    ConfigurationNode specNode = getSpecNode(os);
+    boolean bAdd = (specNode == null);
+    if (bAdd) {
+      specNode = new SpecificationNode(HDFSOutputConstant.PARAM_ROOTPATH);
+    }
+    HDFSOutputSpecs.contextToSpecNode(variableContext, specNode);
+    if (bAdd) {
+      os.addChild(os.getChildCount(), specNode);
+    }
+
+    return null;
+  }
+
+  /** View specification.
+   * This method is called in the body section of a job's view page.  Its purpose is to present the output specification information to the user.
+   * The coder can presume that the HTML that is output from this configuration will be within appropriate <html> and <body> tags.
+   *@param out is the output to which any HTML should be sent.
+   *@param os is the current output specification for this job.
+   */
+  @Override
+  public void viewSpecification(IHTTPOutput out, Locale locale, OutputSpecification os) throws ManifoldCFException, IOException {
+    outputResource(VIEW_SPECIFICATION_HTML, out, locale, getSpecParameters(os), null);
+  }
+
+  /**
+   * @param os
+   * @return
+   */
+  final private SpecificationNode getSpecNode(OutputSpecification os)
+  {
+    int l = os.getChildCount();
+    for (int i = 0; i < l; i++) {
+      SpecificationNode node = os.getChild(i);
+      if (node.getType().equals(HDFSOutputConstant.PARAM_ROOTPATH)) {
+        return node;
+      }
+    }
+    return null;
+  }
+
+  /**
+   * @param os
+   * @return
+   * @throws ManifoldCFException
+   */
+  final private HDFSOutputSpecs getSpecParameters(OutputSpecification os) throws ManifoldCFException {
+    return new HDFSOutputSpecs(getSpecNode(os));
+  }
+
+  /**
+   * @param configParams
+   * @return
+   */
+  final private HDFSOutputConfig getConfigParameters(ConfigParams configParams) {
+    if (configParams == null)
+      configParams = getConfiguration();
+    return new HDFSOutputConfig(configParams);
+  }
+
+  /** Read the content of a resource, replace the variable ${PARAMNAME} with the
+   * value and copy it to the out.
+   * 
+   * @param resName
+   * @param out
+   * @throws ManifoldCFException */
+  private static void outputResource(String resName, IHTTPOutput out, Locale locale, HDFSOutputParam params, String tabName) throws ManifoldCFException {
+    Map<String,String> paramMap = null;
+    if (params != null) {
+      paramMap = params.buildMap();
+      if (tabName != null) {
+        paramMap.put("TabName", tabName);
+      }
+    }
+    Messages.outputResourceWithVelocity(out, locale, resName, paramMap, true);
+  }
+
+  /**
+   * @param documentURI
+   * @return
+   * @throws URISyntaxException
+   * @throws NullPointerException
+   */
+  final private String documentURItoFilePath(String documentURI) throws URISyntaxException, NullPointerException {
+    StringBuffer path = new StringBuffer();
+    URI uri = null;
+
+    uri = new URI(documentURI);
+
+    if (uri.getScheme() != null) {
+      path.append(uri.getScheme());
+      path.append("/");
+    }
+
+    if (uri.getHost() != null) {
+      path.append(uri.getHost());
+      if (uri.getPort() != -1) {
+        path.append(":");
+        path.append(uri.getPort());
+      }
+      if (uri.getRawPath() != null) {
+        if (uri.getRawPath().length() == 0) {
+          path.append("/");
+        } else if (uri.getRawPath().equals("/")) {
+          path.append(uri.getRawPath());
+        } else {
+          for (String name : uri.getRawPath().split("/")) {
+            if (name.length() > 0) {
+              path.append("/");
+              path.append(name);
+            }
+          }
+        }
+      }
+      if (uri.getRawQuery() != null) {
+        path.append("?");
+        path.append(uri.getRawQuery());
+      }
+    } else {
+      if (uri.getRawSchemeSpecificPart() != null) {
+        for (String name : uri.getRawSchemeSpecificPart().split("/")) {
+          if (name.length() > 0) {
+            path.append("/");
+            path.append(name);
+          }
+        }
+      }
+    }
+
+    if (path.toString().endsWith("/")) {
+      path.append(".content");
+    }
+    return path.toString();
+  }
+}

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,38 @@
+/* $Id: FileOutputConstant.java 991374 2013-05-31 23:01:08Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.manifoldcf.agents.output.hdfs;
+
+
+/** Parameters and output data for File output connector.
+ */
+public class HDFSOutputConstant
+{
+  public static final String _rcsid = "@(#)$Id: SolrConfig.java 991374 2010-08-31 22:32:08Z minoru $";
+
+  // Configuration parameters
+
+  /** Name node */
+  public static final String PARAM_NAMENODE = "namenode";
+
+  /** User */
+  public static final String PARAM_USER = "user";
+
+  /** Root path */
+  public static final String PARAM_ROOTPATH = "rootpath";
+}

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConstant.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Added: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java?rev=1495913&view=auto
==============================================================================
--- manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java (added)
+++ manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java Mon Jun 24 03:26:56 2013
@@ -0,0 +1,60 @@
+/* $Id: FileOutputParam.java 1299512 2013-05-31 22:59:38Z minoru $ */
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.manifoldcf.agents.output.hdfs;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.manifoldcf.agents.output.hdfs.HDFSOutputParam.ParameterEnum;
+
+/** 
+ * Parameters data for the elasticsearch output connector.
+ */
+public class HDFSOutputParam extends HashMap<ParameterEnum, String>
+{
+
+  /** Parameters constants */
+  public enum ParameterEnum {
+    NAMENODE("hdfs://localhost:9000"),
+    USER(""),
+    ROOTPATH("/tmp");
+
+    final protected String defaultValue;
+
+    private ParameterEnum(String defaultValue) {
+      this.defaultValue = defaultValue;
+    }
+  }
+
+  private static final long serialVersionUID = -140994685772720029L;
+
+  protected HDFSOutputParam(ParameterEnum[] params) {
+    super(params.length);
+  }
+
+  final public Map<String, String> buildMap() {
+    Map<String, String> rval = new HashMap<String, String>();
+    for (Map.Entry<ParameterEnum, String> entry : this.entrySet()) {
+      rval.put(entry.getKey().name(), entry.getValue());
+    }
+    return rval;
+  }
+
+}

Propchange: manifoldcf/branches/CONNECTORS-728/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputParam.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message