Author: tejasp
Date: Tue May 21 01:31:44 2013
New Revision: 1484638
URL: http://svn.apache.org/r1484638
Log:
NUTCH-1513 Support Robots.txt for Ftp urls
Added:
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1484638&r1=1484637&r2=1484638&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue May 21 01:31:44 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1513 Support Robots.txt for Ftp urls (tejas)
+
* NUTCH-1249 and NUTCH-1275 : Resolve all issues flagged up by adding javac -Xlint argument
(tejasp)
* NUTCH-1053 Parsing of RSS feeds fails (tejasp)
Modified: nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=1484638&r1=1484637&r2=1484638&view=diff
==============================================================================
--- nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
(original)
+++ nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
Tue May 21 01:31:44 2013
@@ -131,8 +131,6 @@ public abstract class HttpBase implement
public Configuration getConf() {
return this.conf;
}
-
-
public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) {
Modified: nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=1484638&r1=1484637&r2=1484638&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java (original)
+++ nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java Tue
May 21 01:31:44 2013
@@ -29,11 +29,9 @@ import org.apache.nutch.net.protocols.Re
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.RobotRulesParser;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatus;
-
import crawlercommons.robots.BaseRobotRules;
import java.net.URL;
@@ -48,8 +46,6 @@ import java.io.IOException;
* {@code ftp.server.timeout}, {@code ftp.password},
* {@code ftp.keep.connection} and {@code ftp.follow.talk}.
* For details see "FTP properties" section in {@code nutch-default.xml}.
- *
- * @author John Xing
*/
public class Ftp implements Protocol {
@@ -84,9 +80,11 @@ public class Ftp implements Protocol {
private Configuration conf;
+ private FtpRobotRulesParser robots = null;
// constructor
public Ftp() {
+ robots = new FtpRobotRulesParser();
}
/** Set the timeout. */
@@ -240,6 +238,7 @@ public class Ftp implements Protocol {
this.serverTimeout = conf.getInt("ftp.server.timeout", 60 * 1000);
this.keepConnection = conf.getBoolean("ftp.keep.connection", false);
this.followTalk = conf.getBoolean("ftp.follow.talk", false);
+ this.robots.setConf(conf);
}
/**
@@ -250,12 +249,10 @@ public class Ftp implements Protocol {
}
/**
- * Currently, no robots parsing is done for ftp protocol
- * and this returns a set of empty rules which will allow every url.
- * There a jira logged for the same NUTCH-1513
+ * Get the robots rules for a given url
*/
public BaseRobotRules getRobotRules(Text url, CrawlDatum datum) {
- return RobotRulesParser.EMPTY_RULES;
+ return robots.getRobotRulesSet(this, url);
}
public int getBufferSize() {
Added: nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java?rev=1484638&view=auto
==============================================================================
--- nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
(added)
+++ nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
Tue May 21 01:31:44 2013
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.protocol.ftp;
+
+import java.net.URL;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.protocol.Protocol;
+import org.apache.nutch.protocol.ProtocolOutput;
+import org.apache.nutch.protocol.ProtocolStatus;
+import org.apache.nutch.protocol.RobotRulesParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
+
+/**
+ * This class is used for parsing robots for urls belonging to FTP protocol.
+ * It extends the generic {@link RobotRulesParser} class and contains
+ * Ftp protocol specific implementation for obtaining the robots file.
+ */
+public class FtpRobotRulesParser extends RobotRulesParser {
+
+ private static final String CONTENT_TYPE = "text/plain";
+ public static final Logger LOG = LoggerFactory.getLogger(FtpRobotRulesParser.class);
+
+ FtpRobotRulesParser() { }
+
+ public FtpRobotRulesParser(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * The hosts for which the caching of robots rules is yet to be done,
+ * it sends a Ftp request to the host corresponding to the {@link URL}
+ * passed, gets robots file, parses the rules and caches the rules object
+ * to avoid re-work in future.
+ *
+ * @param ftp The {@link Protocol} object
+ * @param url URL
+ *
+ * @return robotRules A {@link BaseRobotRules} object for the rules
+ */
+ public BaseRobotRules getRobotRulesSet(Protocol ftp, URL url) {
+
+ String protocol = url.getProtocol().toLowerCase(); // normalize to lower case
+ String host = url.getHost().toLowerCase(); // normalize to lower case
+
+ BaseRobotRules robotRules = (SimpleRobotRules) CACHE.get(protocol + ":" + host);
+
+ boolean cacheRule = true;
+
+ if (robotRules == null) { // cache miss
+ if (LOG.isTraceEnabled())
+ LOG.trace("cache miss " + url);
+
+ try {
+ Text robotsUrl = new Text(new URL(url, "/robots.txt").toString());
+ ProtocolOutput output = ((Ftp)ftp).getProtocolOutput(robotsUrl, new CrawlDatum());
+ ProtocolStatus status = output.getStatus();
+
+ if (status.getCode() == ProtocolStatus.SUCCESS) {
+ robotRules = parseRules(url.toString(), output.getContent().getContent(),
+ CONTENT_TYPE, agentNames);
+ } else {
+ robotRules = EMPTY_RULES; // use default rules
+ }
+ } catch (Throwable t) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Couldn't get robots.txt for " + url + ": " + t.toString());
+ }
+ cacheRule = false;
+ robotRules = EMPTY_RULES;
+ }
+
+ if (cacheRule)
+ CACHE.put(protocol + ":" + host, robotRules); // cache rules for host
+ }
+ return robotRules;
+ }
+}
Author: tejasp
Date: Tue May 21 01:29:55 2013
New Revision: 1484637
URL: http://svn.apache.org/r1484637
Log:
NUTCH-1513 Support Robots.txt for Ftp urls
Added:
nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1484637&r1=1484636&r2=1484637&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Tue May 21 01:29:55 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1513 Support Robots.txt for Ftp urls (tejasp)
+
* NUTCH-1053 Parsing of RSS feeds fails (tejasp)
* NUTCH-1563 FetchSchedule#getFields is never used by GeneratorJob (Feng)
Modified: nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=1484637&r1=1484636&r2=1484637&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
(original)
+++ nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
Tue May 21 01:29:55 2013
@@ -28,7 +28,6 @@ import org.apache.commons.net.ftp.FTPFil
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.RobotRulesParser;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
@@ -46,8 +45,6 @@ import crawlercommons.robots.BaseRobotRu
* {@code ftp.server.timeout}, {@code ftp.password},
* {@code ftp.keep.connection} and {@code ftp.follow.talk}.
* For details see "FTP properties" section in {@code nutch-default.xml}.
- *
- * @author John Xing
*/
public class Ftp implements Protocol {
@@ -89,8 +86,11 @@ public class Ftp implements Protocol {
private Configuration conf;
+ private FtpRobotRulesParser robots = null;
+
// constructor
public Ftp() {
+ robots = new FtpRobotRulesParser();
}
/** Set the timeout. */
@@ -179,6 +179,7 @@ public class Ftp implements Protocol {
this.serverTimeout = conf.getInt("ftp.server.timeout", 60 * 1000);
this.keepConnection = conf.getBoolean("ftp.keep.connection", false);
this.followTalk = conf.getBoolean("ftp.follow.talk", false);
+ this.robots.setConf(conf);
}
/**
@@ -257,11 +258,9 @@ public class Ftp implements Protocol {
}
/**
- * Currently, no robots parsing is done for ftp protocol
- * and this returns a set of empty rules which will allow every url.
- * There a jira logged for the same NUTCH-1513
+ * Get the robots rules for a given url
*/
public BaseRobotRules getRobotRules(String url, WebPage page) {
- return RobotRulesParser.EMPTY_RULES;
+ return robots.getRobotRulesSet(this, url);
}
}
Added: nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java?rev=1484637&view=auto
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
(added)
+++ nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpRobotRulesParser.java
Tue May 21 01:29:55 2013
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.protocol.ftp;
+
+import java.net.URL;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.protocol.Protocol;
+import org.apache.nutch.protocol.ProtocolOutput;
+import org.apache.nutch.protocol.ProtocolStatusCodes;
+import org.apache.nutch.protocol.RobotRulesParser;
+import org.apache.nutch.storage.WebPage;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
+
+/**
+ * This class is used for parsing robots for urls belonging to FTP protocol.
+ * It extends the generic {@link RobotRulesParser} class and contains
+ * Ftp protocol specific implementation for obtaining the robots file.
+ */
+public class FtpRobotRulesParser extends RobotRulesParser {
+
+ private static final String CONTENT_TYPE = "text/plain";
+ public static final Logger LOG = LoggerFactory.getLogger(FtpRobotRulesParser.class);
+
+ FtpRobotRulesParser() { }
+
+ public FtpRobotRulesParser(Configuration conf) {
+ super(conf);
+ }
+
+ /**
+ * The hosts for which the caching of robots rules is yet to be done,
+ * it sends a Ftp request to the host corresponding to the {@link URL}
+ * passed, gets robots file, parses the rules and caches the rules object
+ * to avoid re-work in future.
+ *
+ * @param ftp The {@link Protocol} object
+ * @param url URL
+ *
+ * @return robotRules A {@link BaseRobotRules} object for the rules
+ */
+ public BaseRobotRules getRobotRulesSet(Protocol ftp, URL url) {
+
+ String protocol = url.getProtocol().toLowerCase(); // normalize to lower case
+ String host = url.getHost().toLowerCase(); // normalize to lower case
+
+ BaseRobotRules robotRules = (SimpleRobotRules) CACHE.get(protocol + ":" + host);
+
+ boolean cacheRule = true;
+
+ if (robotRules == null) { // cache miss
+
+ if (LOG.isTraceEnabled())
+ LOG.trace("cache miss " + url);
+
+ try {
+ String robotsUrl = new URL(url, "/robots.txt").toString();
+ ProtocolOutput output = ((Ftp)ftp).getProtocolOutput(robotsUrl, new WebPage());
+ int statusCode = output.getStatus().getCode();
+
+ if (statusCode == ProtocolStatusCodes.SUCCESS) {
+ robotRules = parseRules(url.toString(), output.getContent().getContent(),
+ CONTENT_TYPE, agentNames);
+ } else {
+ robotRules = EMPTY_RULES; // use default rules
+ }
+ } catch (Throwable t) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Couldn't get robots.txt for " + url + ": " + t.toString());
+ }
+ cacheRule = false;
+ robotRules = EMPTY_RULES;
+ }
+
+ if (cacheRule)
+ CACHE.put(protocol + ":" + host, robotRules); // cache rules for host
+ }
+ return robotRules;
+ }
+}
Modified: nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java (original)
+++ nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java Tue
May 21 01:19:26 2013
@@ -42,8 +42,6 @@ import com.anotherbigidea.io.InStream;
/**
* Parser for Flash SWF files. Loosely based on the sample in JavaSWF
* distribution.
- *
- * @author Andrzej Bialecki
*/
public class SWFParser implements Parser {
public static final Logger LOG = LoggerFactory.getLogger("org.apache.nutch.parse.swf");
@@ -63,7 +61,7 @@ public class SWFParser implements Parser
public ParseResult getParse(Content content) {
String text = null;
- Vector outlinks = new Vector();
+ Vector<Outlink> outlinks = new Vector<Outlink>();
try {
@@ -120,6 +118,7 @@ public class SWFParser implements Parser
byte[] buf = new byte[in.available()];
in.read(buf);
+ in.close();
SWFParser parser = new SWFParser();
ParseResult parseResult = parser.getParse(new Content("file:" + args[0], "file:" + args[0],
buf, "application/x-shockwave-flash",
@@ -153,13 +152,13 @@ class ExtractText extends SWFTagTypesImp
* character codes for the correspnding font glyphs (An empty array denotes a
* System Font).
*/
- protected HashMap fontCodes = new HashMap();
+ protected HashMap<Integer, int[]> fontCodes = new HashMap<Integer, int[]>();
- public ArrayList strings = new ArrayList();
+ public ArrayList<String> strings = new ArrayList<String>();
- public HashSet actionStrings = new HashSet();
+ public HashSet<String> actionStrings = new HashSet<String>();
- public ArrayList urls = new ArrayList();
+ public ArrayList<String> urls = new ArrayList<String>();
public ExtractText() {
super(null);
@@ -167,7 +166,7 @@ class ExtractText extends SWFTagTypesImp
public String getText() {
StringBuffer res = new StringBuffer();
- Iterator it = strings.iterator();
+ Iterator<String> it = strings.iterator();
while (it.hasNext()) {
if (res.length() > 0) res.append(' ');
res.append(it.next());
@@ -189,7 +188,7 @@ class ExtractText extends SWFTagTypesImp
public String[] getUrls() {
String[] res = new String[urls.size()];
int i = 0;
- Iterator it = urls.iterator();
+ Iterator<String> it = urls.iterator();
while (it.hasNext()) {
res[i] = (String) it.next();
i++;
@@ -350,26 +349,23 @@ class ExtractText extends SWFTagTypesImp
* ActionScript parser. This parser tries to extract free text embedded inside
* the script, but without polluting it too much with names of variables,
* methods, etc. Not ideal, but it works.
- *
- * @author Andrzej Bialecki
*/
class NutchSWFActions extends SWFActionBlockImpl implements SWFActions {
- private HashSet strings = null;
+ private HashSet<String> strings = null;
- private ArrayList urls = null;
+ private ArrayList<String> urls = null;
String[] dict = null;
- Stack stack = null;
+ Stack<Object> stack = null;
- public NutchSWFActions(HashSet strings, ArrayList urls) {
+ public NutchSWFActions(HashSet<String> strings, ArrayList<String> urls) {
this.strings = strings;
this.urls = urls;
stack = new SmallStack(100, strings);
}
public void lookupTable(String[] values) throws IOException {
- // System.out.println("-lookupTable: " + values.length);
for (int i = 0; i < values.length; i++) {
if (!strings.contains(values[i])) strings.add(values[i]);
}
@@ -378,7 +374,6 @@ class NutchSWFActions extends SWFActionB
}
public void defineLocal() throws IOException {
- // System.out.println("-defineLocal");
stack.pop();
super.defineLocal();
}
@@ -398,69 +393,58 @@ class NutchSWFActions extends SWFActionB
}
public SWFActionBlock.TryCatchFinally _try(String var) throws IOException {
- // System.out.println("_try: var=" + var);
// stack.push(var);
strings.remove(var);
return super._try(var);
}
public void comment(String var) throws IOException {
- // System.out.println("-comment: var=" + var);
// stack.push(var);
strings.remove(var);
super.comment(var);
}
public void goToFrame(String var) throws IOException {
- // System.out.println("-goToFrame: var=" + var);
stack.push(var);
strings.remove(var);
super.gotoFrame(var);
}
public void ifJump(String var) throws IOException {
- // System.out.println("-ifJump: var=" + var);
strings.remove(var);
super.ifJump(var);
}
public void jump(String var) throws IOException {
- // System.out.println("-jump: var=" + var);
strings.remove(var);
super.jump(var);
}
public void jumpLabel(String var) throws IOException {
- // System.out.println("-jumpLabel: var=" + var);
strings.remove(var);
super.jumpLabel(var);
}
public void lookup(int var) throws IOException {
- // System.out.println("-lookup: var=" + var);
if (dict != null && var >= 0 && var < dict.length) {
- // System.out.println(" push " + dict[var]);
stack.push(dict[var]);
}
super.lookup(var);
}
public void push(String var) throws IOException {
- // System.out.println("-push: var=" + var);
stack.push(var);
strings.remove(var);
super.push(var);
}
public void setTarget(String var) throws IOException {
- // System.out.println("-setTarget: var=" + var);
stack.push(var);
strings.remove(var);
super.setTarget(var);
}
public SWFActionBlock startFunction(String var, String[] params) throws IOException {
- // System.out.println("-startFunction1: var=" + var);
stack.push(var);
strings.remove(var);
if (params != null) {
@@ -472,7 +456,6 @@ class NutchSWFActions extends SWFActionB
}
public SWFActionBlock startFunction2(String var, int arg1, int arg2, String[] params, int[]
arg3) throws IOException {
- // System.out.println("-startFunction2: var=" + var);
stack.push(var);
strings.remove(var);
if (params != null) {
@@ -484,74 +467,61 @@ class NutchSWFActions extends SWFActionB
}
public void waitForFrame(int num, String var) throws IOException {
- // System.out.println("-waitForFrame: var=" + var);
stack.push(var);
strings.remove(var);
super.waitForFrame(num, var);
}
public void waitForFrame(String var) throws IOException {
- // System.out.println("-waitForFrame: var=" + var);
stack.push(var);
strings.remove(var);
super.waitForFrame(var);
}
public void done() throws IOException {
- // System.out.println("-done");
while (stack.size() > 0) {
strings.remove(stack.pop());
}
}
public SWFActionBlock start(int arg0, int arg1) throws IOException {
- // System.out.println("-start");
return this;
}
public SWFActionBlock start(int arg0) throws IOException {
- // System.out.println("-start");
return this;
}
public void add() throws IOException {
- // System.out.println("-add");
super.add();
}
public void asciiToChar() throws IOException {
- // System.out.println("-asciitochar");
super.asciiToChar();
}
public void asciiToCharMB() throws IOException {
- // System.out.println("-asciitocharMB");
super.asciiToCharMB();
}
public void push(int var) throws IOException {
- // System.out.println("-push(int)");
if (dict != null && var >= 0 && var < dict.length) {
- // System.out.println(" push " + dict[var]);
stack.push(dict[var]);
}
super.push(var);
}
public void callFunction() throws IOException {
- // System.out.println("-callFunction");
strings.remove(stack.pop());
super.callFunction();
}
public void callMethod() throws IOException {
- // System.out.println("-callMethod");
strings.remove(stack.pop());
super.callMethod();
}
public void getMember() throws IOException {
- // System.out.println("-getMember");
// 0: name
String val = (String) stack.pop();
strings.remove(val);
@@ -560,116 +530,97 @@ class NutchSWFActions extends SWFActionB
public void setMember() throws IOException {
// 0: value -1: name
- String val = (String) stack.pop();
+ stack.pop(); // value
String name = (String) stack.pop();
- // System.out.println("-setMember: name=" + name + ", val=" + val);
strings.remove(name);
super.setMember();
}
public void setProperty() throws IOException {
- // System.out.println("-setProperty");
super.setProperty();
}
public void setVariable() throws IOException {
- // System.out.println("-setVariable");
super.setVariable();
}
public void call() throws IOException {
- // System.out.println("-call");
strings.remove(stack.pop());
super.call();
}
public void setTarget() throws IOException {
- // System.out.println("-setTarget");
strings.remove(stack.pop());
super.setTarget();
}
public void pop() throws IOException {
- // System.out.println("-pop");
strings.remove(stack.pop());
super.pop();
}
public void push(boolean arg0) throws IOException {
- // System.out.println("-push(b)");
stack.push("" + arg0);
super.push(arg0);
}
public void push(double arg0) throws IOException {
- // System.out.println("-push(d)");
stack.push("" + arg0);
super.push(arg0);
}
public void push(float arg0) throws IOException {
- // System.out.println("-push(f)");
stack.push("" + arg0);
super.push(arg0);
}
public void pushNull() throws IOException {
- // System.out.println("-push(null)");
stack.push("");
super.pushNull();
}
public void pushRegister(int arg0) throws IOException {
- // System.out.println("-push(reg)");
stack.push("" + arg0);
super.pushRegister(arg0);
}
public void pushUndefined() throws IOException {
- // System.out.println("-push(undef)");
stack.push("???");
super.pushUndefined();
}
public void getProperty() throws IOException {
- // System.out.println("-getProperty");
stack.pop();
super.getProperty();
}
public void getVariable() throws IOException {
- // System.out.println("-getVariable");
strings.remove(stack.pop());
super.getVariable();
}
public void gotoFrame(boolean arg0) throws IOException {
- // System.out.println("-gotoFrame(b)");
stack.push("" + arg0);
super.gotoFrame(arg0);
}
public void gotoFrame(int arg0) throws IOException {
- // System.out.println("-gotoFrame(int)");
stack.push("" + arg0);
super.gotoFrame(arg0);
}
public void gotoFrame(String arg0) throws IOException {
- // System.out.println("-gotoFrame(string)");
stack.push("" + arg0);
strings.remove(arg0);
super.gotoFrame(arg0);
}
public void newObject() throws IOException {
- // System.out.println("-newObject");
stack.pop();
super.newObject();
}
public SWFActionBlock startWith() throws IOException {
- // System.out.println("-startWith");
return this;
}
@@ -678,13 +629,15 @@ class NutchSWFActions extends SWFActionB
/*
* Small bottom-less stack.
*/
-class SmallStack extends Stack {
+class SmallStack extends Stack<Object> {
+
+ private static final long serialVersionUID = 1L;
private int maxSize;
- private HashSet strings = null;
+ private HashSet<String> strings = null;
- public SmallStack(int maxSize, HashSet strings) {
+ public SmallStack(int maxSize, HashSet<String> strings) {
this.maxSize = maxSize;
this.strings = strings;
}
Modified: nutch/trunk/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java
(original)
+++ nutch/trunk/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java
Tue May 21 01:19:26 2013
@@ -32,13 +32,12 @@ import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
+import org.mortbay.log.Log;
import junit.framework.TestCase;
/**
* Unit tests for SWFParser.
- *
- * @author Andrzej Bialecki
*/
public class TestSWFParser extends TestCase {
@@ -48,7 +47,6 @@ public class TestSWFParser extends TestC
private String[] sampleFiles = new String[]{"test1.swf", "test2.swf", "test3.swf"};
private String[] sampleTexts = new String[]{"test1.txt", "test2.txt", "test3.txt"};
- private String[] texts = new String[sampleTexts.length];
public TestSWFParser(String name) {
super(name);
@@ -94,5 +92,4 @@ public class TestSWFParser extends TestC
assertTrue(sampleTexts[i].equals(text));
}
}
-
}
Modified: nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMBuilder.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMBuilder.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMBuilder.java
(original)
+++ nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMBuilder.java
Tue May 21 01:19:26 2013
@@ -58,7 +58,7 @@ class DOMBuilder
public DocumentFragment m_docFrag = null;
/** Vector of element nodes */
- protected Stack m_elemStack = new Stack();
+ protected Stack<Element> m_elemStack = new Stack<Element>();
/**
* DOMBuilder instance constructor... it will add the DOM nodes
Modified: nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
(original)
+++ nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
Tue May 21 01:19:26 2013
@@ -59,7 +59,8 @@ public class TikaParser implements org.a
private HtmlParseFilters htmlParseFilters;
private String cachingPolicy;
- public ParseResult getParse(Content content) {
+ @SuppressWarnings("deprecation")
+ public ParseResult getParse(Content content) {
String mimeType = content.getContentType();
URL base;
Modified: nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java (original)
+++ nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java Tue
May 21 01:19:26 2013
@@ -18,15 +18,12 @@
package org.apache.nutch.parse.zip;
import java.io.ByteArrayInputStream;
-import java.io.InputStream;
-import java.util.Properties;
import java.util.ArrayList;
import java.util.List;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.parse.ParseData;
@@ -40,8 +37,6 @@ import org.apache.hadoop.conf.Configurat
/**
* ZipParser class based on MSPowerPointParser class by Stephan Strittmatter.
* Nutch parse plugin for zip files - Content Type : application/zip
- *
- * @author Rohit Kulkarni & Ashish Vaidya
*/
public class ZipParser implements Parser {
@@ -57,17 +52,13 @@ public class ZipParser implements Parser
String resultText = null;
String resultTitle = null;
Outlink[] outlinks = null;
- List outLinksList = new ArrayList();
- Properties properties = null;
+ List<Outlink> outLinksList = new ArrayList<Outlink>();
try {
final String contentLen = content.getMetadata().get(Response.CONTENT_LENGTH);
final int len = Integer.parseInt(contentLen);
if (LOG.isDebugEnabled()) { LOG.debug("ziplen: " + len); }
final byte[] contentInBytes = content.getContent();
- final ByteArrayInputStream bainput = new ByteArrayInputStream(
- contentInBytes);
- final InputStream input = bainput;
if (contentLen != null && contentInBytes.length != len) {
return new ParseStatus(ParseStatus.FAILED,
Modified: nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
(original)
+++ nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
Tue May 21 01:19:26 2013
@@ -77,9 +77,9 @@ public class Client extends FTP
private int __dataTimeout;
private int __passivePort;
private String __passiveHost;
- private int __fileType, __fileFormat;
+// private int __fileType, __fileFormat;
private boolean __remoteVerificationEnabled;
- private FTPFileEntryParser __entryParser;
+// private FTPFileEntryParser __entryParser;
private String __systemName;
// constructor
@@ -95,10 +95,10 @@ public class Client extends FTP
{
__passiveHost = null;
__passivePort = -1;
- __fileType = FTP.ASCII_FILE_TYPE;
- __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
__systemName = null;
- __entryParser = null;
+// __fileType = FTP.ASCII_FILE_TYPE;
+// __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
+// __entryParser = null;
}
// parse reply for pass()
@@ -315,7 +315,7 @@ public class Client extends FTP
}
// retrieve list reply for path
- public void retrieveList(String path, List entries, int limit,
+ public void retrieveList(String path, List<FTPFile> entries, int limit,
FTPFileEntryParser parser)
throws IOException,
FtpExceptionCanNotHaveDataConnection,
@@ -331,7 +331,7 @@ public class Client extends FTP
new BufferedReader(new InputStreamReader(socket.getInputStream()));
// force-close data channel socket, when download limit is reached
- boolean mandatory_close = false;
+// boolean mandatory_close = false;
//List entries = new LinkedList();
int count = 0;
@@ -348,7 +348,7 @@ public class Client extends FTP
// impose download limit if limit >= 0, otherwise no limit
// here, cut off is up to the line when total bytes is just over limit
if (limit >= 0 && count > limit) {
- mandatory_close = true;
+// mandatory_close = true;
break;
}
line = parser.readNextEntry(reader);
@@ -403,7 +403,7 @@ public class Client extends FTP
// fixme, should we instruct server here for binary file type?
// force-close data channel socket
- boolean mandatory_close = false;
+ // boolean mandatory_close = false;
int len; int count = 0;
byte[] buf =
@@ -414,7 +414,7 @@ public class Client extends FTP
// here, cut off is exactly of limit bytes
if (limit >= 0 && count > limit) {
os.write(buf,0,len-(count-limit));
- mandatory_close = true;
+ // mandatory_close = true;
break;
}
os.write(buf,0,len);
@@ -502,8 +502,8 @@ public class Client extends FTP
{
if (FTPReply.isPositiveCompletion(type(fileType)))
{
- __fileType = fileType;
- __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;
+/* __fileType = fileType;
+ __fileFormat = FTP.NON_PRINT_TEXT_FORMAT;*/
return true;
}
return false;
Modified: nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java (original)
+++ nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java Tue
May 21 01:19:26 2013
@@ -55,7 +55,7 @@ public class Ftp implements Protocol {
public static final Logger LOG = LoggerFactory.getLogger(Ftp.class);
- static final int BUFFER_SIZE = 16384; // 16*1024 = 16384
+ private static final int BUFFER_SIZE = 16384; // 16*1024 = 16384
static final int MAX_REDIRECTS = 5;
@@ -257,5 +257,9 @@ public class Ftp implements Protocol {
public BaseRobotRules getRobotRules(Text url, CrawlDatum datum) {
return RobotRulesParser.EMPTY_RULES;
}
+
+ public int getBufferSize() {
+ return BUFFER_SIZE;
+ }
}
Modified: nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
(original)
+++ nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
Tue May 21 01:19:26 2013
@@ -17,11 +17,9 @@
package org.apache.nutch.protocol.ftp;
-
import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPFile;
import org.apache.commons.net.ftp.FTPReply;
-
import org.apache.commons.net.ftp.parser.DefaultFTPFileEntryParserFactory;
import org.apache.commons.net.ftp.parser.ParserInitializationException;
@@ -42,8 +40,7 @@ import java.util.LinkedList;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-
-/************************************
+/**
* FtpResponse.java mimics ftp replies as http response.
* It tries its best to follow http's way for headers, response codes
* as well as exceptions.
@@ -53,9 +50,7 @@ import java.io.IOException;
* and some important commons-net exceptions passed by Client.java
* must have been properly dealt with. They'd better not be leaked
* to the caller of this class.
- *
- * @author John Xing
- ***********************************/
+ */
public class FtpResponse {
private String orig;
@@ -146,7 +141,7 @@ public class FtpResponse {
// follow ftp talk?
if (ftp.followTalk)
ftp.client.addProtocolCommandListener(
- new PrintCommandListener(ftp.LOG));
+ new PrintCommandListener(Ftp.LOG));
}
// quit from previous site if at a different site now
@@ -284,8 +279,8 @@ public class FtpResponse {
}
} catch (Exception e) {
- if (ftp.LOG.isWarnEnabled()) {
- ftp.LOG.warn("Error: ", e);
+ if (Ftp.LOG.isWarnEnabled()) {
+ Ftp.LOG.warn("Error: ", e);
}
// for any un-foreseen exception (run time exception or not),
// do ultimate clean and leave ftp.client for garbage collection
@@ -312,11 +307,11 @@ public class FtpResponse {
throws IOException {
ByteArrayOutputStream os = null;
- List list = null;
+ List<FTPFile> list = null;
try {
// first get its possible attributes
- list = new LinkedList();
+ list = new LinkedList<FTPFile>();
ftp.client.retrieveList(path, list, ftp.maxContentLength, ftp.parser);
FTPFile ftpFile = (FTPFile) list.get(0);
@@ -329,7 +324,7 @@ public class FtpResponse {
code = 304;
return;
}
- os = new ByteArrayOutputStream(ftp.BUFFER_SIZE);
+ os = new ByteArrayOutputStream(ftp.getBufferSize());
ftp.client.retrieveFile(path, os, ftp.maxContentLength);
this.content = os.toByteArray();
@@ -414,7 +409,7 @@ public class FtpResponse {
// get ftp dir list as http response
private void getDirAsHttpResponse(String path, long lastModified)
throws IOException {
- List list = new LinkedList();
+ List<FTPFile> list = new LinkedList<FTPFile>();
try {
@@ -482,7 +477,7 @@ public class FtpResponse {
}
// generate html page from ftp dir list
- private byte[] list2html(List list, String path, boolean includeDotDot) {
+ private byte[] list2html(List<FTPFile> list, String path, boolean includeDotDot)
{
//StringBuffer x = new StringBuffer("<!doctype html public \"-//ietf//dtd html//en\"><html><head>");
StringBuffer x = new StringBuffer("<html><head>");
Modified: nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
(original)
+++ nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Tue May 21 01:19:26 2013
@@ -42,6 +42,8 @@ import org.apache.commons.httpclient.NTC
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.httpclient.protocol.Protocol;
+import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
+import org.apache.commons.httpclient.protocol.SSLProtocolSocketFactory;
// Nutch imports
import org.apache.nutch.crawl.CrawlDatum;
@@ -158,8 +160,8 @@ public class Http extends HttpBase {
private void configureClient() {
// Set up an HTTPS socket factory that accepts self-signed certs.
- Protocol https = new Protocol("https",
- new DummySSLProtocolSocketFactory(), 443);
+ ProtocolSocketFactory factory = new SSLProtocolSocketFactory();
+ Protocol https = new Protocol("https", factory, 443);
Protocol.registerProtocol("https", https);
HttpConnectionManagerParams params = connectionManager.getParams();
@@ -174,7 +176,7 @@ public class Http extends HttpBase {
client.getParams().setConnectionManagerTimeout(timeout);
HostConfiguration hostConf = client.getHostConfiguration();
- ArrayList headers = new ArrayList();
+ ArrayList<Header> headers = new ArrayList<Header>();
// Set the User Agent in the header
headers.add(new Header("User-Agent", userAgent));
// prefer English
@@ -199,7 +201,7 @@ public class Http extends HttpBase {
NTCredentials proxyCredentials = new NTCredentials(
this.proxyUsername, this.proxyPassword,
- this.agentHost, this.proxyRealm);
+ Http.agentHost, this.proxyRealm);
client.getState().setProxyCredentials(
proxyAuthScope, proxyCredentials);
Modified: nutch/trunk/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
(original)
+++ nutch/trunk/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
Tue May 21 01:19:26 2013
@@ -23,7 +23,6 @@ import junit.framework.TestCase;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.bio.SocketConnector;
import org.mortbay.jetty.handler.ContextHandler;
-import org.mortbay.jetty.handler.ResourceHandler;
import org.mortbay.jetty.servlet.ServletHandler;
import org.mortbay.jetty.servlet.SessionHandler;
import org.apache.hadoop.conf.Configuration;
@@ -32,8 +31,6 @@ import org.apache.nutch.net.protocols.Re
/**
* Test cases for protocol-httpclient.
- *
- * @author Susam Pal
*/
public class TestProtocolHttpClient extends TestCase {
Modified: nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java
(original)
+++ nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java
Tue May 21 01:19:26 2013
@@ -14,9 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-// $Id$
-
package org.apache.nutch.urlfilter.prefix;
import org.slf4j.Logger;
@@ -79,7 +76,7 @@ public class PrefixURLFilter implements
throws IOException {
BufferedReader in=new BufferedReader(reader);
- List urlprefixes = new ArrayList();
+ List<String> urlprefixes = new ArrayList<String>();
String line;
while((line=in.readLine())!=null) {
Modified: nutch/trunk/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java
(original)
+++ nutch/trunk/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java
Tue May 21 01:19:26 2013
@@ -181,7 +181,7 @@ public class SuffixURLFilter implements
return;
}
BufferedReader in = new BufferedReader(reader);
- List aSuffixes = new ArrayList();
+ List<String> aSuffixes = new ArrayList<String>();
boolean allow = false;
boolean ignore = false;
String line;
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java Tue May 21 01:19:26
2013
@@ -44,9 +44,9 @@ public class TestCrawlDbMerger extends T
url21
};
- TreeSet init1 = new TreeSet();
- TreeSet init2 = new TreeSet();
- HashMap expected = new HashMap();
+ TreeSet<String> init1 = new TreeSet<String>();
+ TreeSet<String> init2 = new TreeSet<String>();
+ HashMap<String, CrawlDatum> expected = new HashMap<String, CrawlDatum>();
CrawlDatum cd1, cd2, cd3;
Configuration conf;
FileSystem fs;
@@ -83,6 +83,7 @@ public class TestCrawlDbMerger extends T
fs.mkdirs(testDir);
}
+ @SuppressWarnings("deprecation")
public void tearDown() {
try {
if (fs.exists(testDir))
@@ -93,6 +94,7 @@ public class TestCrawlDbMerger extends T
} catch (Exception e) { }
}
+ @SuppressWarnings("deprecation")
public void testMerge() throws Exception {
Path crawldb1 = new Path(testDir, "crawldb1");
Path crawldb2 = new Path(testDir, "crawldb2");
@@ -105,11 +107,11 @@ public class TestCrawlDbMerger extends T
LOG.fine("* reading crawldb: " + output);
reader = new CrawlDbReader();
String crawlDb = output.toString();
- Iterator it = expected.keySet().iterator();
+ Iterator<String> it = expected.keySet().iterator();
while (it.hasNext()) {
- String url = (String)it.next();
+ String url = it.next();
LOG.fine("url=" + url);
- CrawlDatum cd = (CrawlDatum)expected.get(url);
+ CrawlDatum cd = expected.get(url);
CrawlDatum res = reader.get(crawlDb, url, conf);
LOG.fine(" -> " + res);
System.out.println("url=" + url);
@@ -123,13 +125,13 @@ public class TestCrawlDbMerger extends T
fs.delete(testDir);
}
- private void createCrawlDb(Configuration config, FileSystem fs, Path crawldb, TreeSet init,
CrawlDatum cd) throws Exception {
+ private void createCrawlDb(Configuration config, FileSystem fs, Path crawldb, TreeSet<String>
init, CrawlDatum cd) throws Exception {
LOG.fine("* creating crawldb: " + crawldb);
Path dir = new Path(crawldb, CrawlDb.CURRENT_NAME);
MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir, "part-00000").toString(),
Text.class, CrawlDatum.class);
- Iterator it = init.iterator();
+ Iterator<String> it = init.iterator();
while (it.hasNext()) {
- String key = (String)it.next();
+ String key = it.next();
writer.append(new Text(key), cd);
}
writer.close();
Modified: nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java Tue May 21 01:19:26
2013
@@ -68,9 +68,9 @@ public class TestLinkDbMerger extends Te
String[] urls20_expected = urls11_expected;
String[] urls21_expected = urls21;
- TreeMap init1 = new TreeMap();
- TreeMap init2 = new TreeMap();
- HashMap expected = new HashMap();
+ TreeMap<String, String[]> init1 = new TreeMap<String, String[]>();
+ TreeMap<String, String[]> init2 = new TreeMap<String, String[]>();
+ HashMap<String, String[]> expected = new HashMap<String, String[]>();
Configuration conf;
Path testDir;
FileSystem fs;
@@ -116,16 +116,16 @@ public class TestLinkDbMerger extends Te
merger.merge(output, new Path[]{linkdb1, linkdb2}, false, false);
LOG.fine("* reading linkdb: " + output);
reader = new LinkDbReader(conf, output);
- Iterator it = expected.keySet().iterator();
+ Iterator<String> it = expected.keySet().iterator();
while (it.hasNext()) {
- String url = (String)it.next();
+ String url = it.next();
LOG.fine("url=" + url);
- String[] vals = (String[])expected.get(url);
+ String[] vals = expected.get(url);
Inlinks inlinks = reader.getInlinks(new Text(url));
// may not be null
assertNotNull(inlinks);
- ArrayList links = new ArrayList();
- Iterator it2 = inlinks.iterator();
+ ArrayList<String> links = new ArrayList<String>();
+ Iterator<?> it2 = inlinks.iterator();
while (it2.hasNext()) {
Inlink in = (Inlink)it2.next();
links.add(in.getFromUrl());
@@ -139,15 +139,15 @@ public class TestLinkDbMerger extends Te
fs.delete(testDir, true);
}
- private void createLinkDb(Configuration config, FileSystem fs, Path linkdb, TreeMap init)
throws Exception {
+ private void createLinkDb(Configuration config, FileSystem fs, Path linkdb, TreeMap<String,
String[]> init) throws Exception {
LOG.fine("* creating linkdb: " + linkdb);
Path dir = new Path(linkdb, LinkDb.CURRENT_NAME);
MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir, "part-00000").toString(),
Text.class, Inlinks.class);
- Iterator it = init.keySet().iterator();
+ Iterator<String> it = init.keySet().iterator();
while (it.hasNext()) {
- String key = (String)it.next();
+ String key = it.next();
Inlinks inlinks = new Inlinks();
- String[] vals = (String[])init.get(key);
+ String[] vals = init.get(key);
for (int i = 0; i < vals.length; i++) {
Inlink in = new Inlink(vals[i], vals[i]);
inlinks.add(in);
Modified: nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java Tue May 21 01:19:26
2013
@@ -35,13 +35,11 @@ import org.apache.nutch.util.NutchJob;
/**
* Unit tests for the plugin system
- *
- * @author joa23
*/
public class TestPluginSystem extends TestCase {
private int fPluginCount;
- private LinkedList fFolders = new LinkedList();
+ private LinkedList<File> fFolders = new LinkedList<File>();
private Configuration conf ;
private PluginRepository repository;
@@ -62,11 +60,10 @@ public class TestPluginSystem extends Te
*/
protected void tearDown() throws Exception {
for (int i = 0; i < fFolders.size(); i++) {
- File folder = (File) fFolders.get(i);
+ File folder = fFolders.get(i);
delete(folder);
folder.delete();
}
-
}
/**
Modified: nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java (original)
+++ nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java Tue May 21 01:19:26
2013
@@ -42,11 +42,11 @@ public class TestSegmentMerger extends T
public void setUp() throws Exception {
conf = NutchConfiguration.create();
fs = FileSystem.get(conf);
- long blkSize = fs.getDefaultBlockSize();
testDir = new Path(conf.get("hadoop.tmp.dir"), "merge-" + System.currentTimeMillis());
seg1 = new Path(testDir, "seg1");
seg2 = new Path(testDir, "seg2");
out = new Path(testDir, "out");
+
// create large parse-text segments
System.err.println("Creating large segment 1...");
DecimalFormat df = new DecimalFormat("0000000");
@@ -55,6 +55,9 @@ public class TestSegmentMerger extends T
MapFile.Writer w = new MapFile.Writer(conf, fs, ptPath.toString(), Text.class, ParseText.class);
long curSize = 0;
countSeg1 = 0;
+ FileStatus fileStatus = fs.getFileStatus(ptPath);
+ long blkSize = fileStatus.getBlockSize();
+
while (curSize < blkSize * 2) {
k.set("seg1-" + df.format(countSeg1));
w.append(k, new ParseText("seg1 text " + countSeg1));
Author: tejasp
Date: Tue May 21 01:19:26 2013
New Revision: 1484634
URL: http://svn.apache.org/r1484634
Log:
NUTCH-1249 and NUTCH-1275 : Resolve all issues flagged up by adding javac -Xlint argument
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/build.xml
nutch/trunk/src/java/org/apache/nutch/crawl/FetchScheduleFactory.java
nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java
nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java
nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java
nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java
nutch/trunk/src/java/org/apache/nutch/crawl/SignatureComparator.java
nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java
nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java
nutch/trunk/src/java/org/apache/nutch/indexer/NutchField.java
nutch/trunk/src/java/org/apache/nutch/metadata/HttpHeaders.java
nutch/trunk/src/java/org/apache/nutch/metadata/Metadata.java
nutch/trunk/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java
nutch/trunk/src/java/org/apache/nutch/net/URLNormalizers.java
nutch/trunk/src/java/org/apache/nutch/parse/HTMLMetaTags.java
nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java
nutch/trunk/src/java/org/apache/nutch/plugin/Extension.java
nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java
nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LoopReader.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/Loops.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java
nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java
nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java
nutch/trunk/src/java/org/apache/nutch/tools/ResolveUrls.java
nutch/trunk/src/java/org/apache/nutch/tools/proxy/SegmentHandler.java
nutch/trunk/src/java/org/apache/nutch/util/GenericWritableConfigurable.java
nutch/trunk/src/java/org/apache/nutch/util/PrefixStringMatcher.java
nutch/trunk/src/java/org/apache/nutch/util/SuffixStringMatcher.java
nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java
nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java
nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMBuilder.java
nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java
nutch/trunk/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java
nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMBuilder.java
nutch/trunk/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Client.java
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
nutch/trunk/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java
nutch/trunk/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java
nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java
nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java
nutch/trunk/src/test/org/apache/nutch/plugin/TestPluginSystem.java
nutch/trunk/src/test/org/apache/nutch/segment/TestSegmentMerger.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue May 21 01:19:26 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1249 and NUTCH-1275 : Resolve all issues flagged up by adding javac -Xlint argument (tejasp)
+
* NUTCH-1053 Parsing of RSS feeds fails (tejasp)
* Added crawler-commons dependency in pom.xml (tejasp)
Modified: nutch/trunk/build.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/build.xml?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/build.xml (original)
+++ nutch/trunk/build.xml Tue May 21 01:19:26 2013
@@ -57,6 +57,10 @@
</fileset>
</path>
+ <presetdef name="javac">
+ <javac includeantruntime="false" />
+ </presetdef>
+
<!-- ====================================================== -->
<!-- Stuff needed by all targets -->
<!-- ====================================================== -->
@@ -94,7 +98,7 @@
target="${javac.version}"
source="${javac.version}"
deprecation="${javac.deprecation}">
- <compilerarg value="-Xlint"/>
+ <compilerarg value="-Xlint:-path"/>
<classpath refid="classpath"/>
</javac>
</target>
@@ -341,7 +345,7 @@
target="${javac.version}"
source="${javac.version}"
deprecation="${javac.deprecation}">
- <compilerarg value="-Xlint"/>
+ <compilerarg value="-Xlint:-path"/>
<classpath refid="test.classpath"/>
</javac>
</target>
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/FetchScheduleFactory.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/FetchScheduleFactory.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/FetchScheduleFactory.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/FetchScheduleFactory.java Tue May 21 01:19:26 2013
@@ -37,7 +37,7 @@ public class FetchScheduleFactory {
if (impl == null) {
try {
LOG.info("Using FetchSchedule impl: " + clazz);
- Class implClass = Class.forName(clazz);
+ Class<?> implClass = Class.forName(clazz);
impl = (FetchSchedule)implClass.newInstance();
impl.setConf(conf);
objectCache.setObject(clazz, impl);
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java Tue May 21 01:19:26 2013
@@ -368,6 +368,7 @@ public class Generator extends Configure
super(Text.class);
}
+ @SuppressWarnings("rawtypes" )
public int compare(WritableComparable a, WritableComparable b) {
Text url1 = (Text) a;
Text url2 = (Text) b;
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java Tue May 21 01:19:26 2013
@@ -60,7 +60,7 @@ public class Injector extends Configured
public static String nutchFixedFetchIntervalMDName = "nutch.fetchInterval.fixed";
/** Normalize and filter injected urls. */
- public static class InjectMapper implements Mapper<WritableComparable, Text, Text, CrawlDatum> {
+ public static class InjectMapper implements Mapper<WritableComparable<?>, Text, Text, CrawlDatum> {
private URLNormalizers urlNormalizers;
private int interval;
private float scoreInjected;
@@ -81,7 +81,7 @@ public class Injector extends Configured
public void close() {}
- public void map(WritableComparable key, Text value,
+ public void map(WritableComparable<?> key, Text value,
OutputCollector<Text, CrawlDatum> output, Reporter reporter)
throws IOException {
String url = value.toString(); // value is line of text
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java Tue May 21 01:19:26 2013
@@ -43,7 +43,7 @@ import java.io.Closeable;
public class LinkDbReader extends Configured implements Tool, Closeable {
public static final Logger LOG = LoggerFactory.getLogger(LinkDbReader.class);
- private static final Partitioner<WritableComparable, Writable> PARTITIONER = new HashPartitioner<WritableComparable, Writable>();
+ private static final Partitioner<WritableComparable<?>, Writable> PARTITIONER = new HashPartitioner<WritableComparable<?>, Writable>();
private FileSystem fs;
private Path directory;
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java Tue May 21 01:19:26 2013
@@ -79,9 +79,9 @@ public class MapWritable implements Writ
private ClassIdEntry fIdFirst;
- private static Map<Class, Byte> CLASS_ID_MAP = new HashMap<Class, Byte>();
+ private static Map<Class<?>, Byte> CLASS_ID_MAP = new HashMap<Class<?>, Byte>();
- private static Map<Byte, Class> ID_CLASS_MAP = new HashMap<Byte, Class>();
+ private static Map<Byte, Class<?>> ID_CLASS_MAP = new HashMap<Byte, Class<?>>();
static {
@@ -101,7 +101,7 @@ public class MapWritable implements Writ
}
- private static void addToMap(Class clazz, Byte byteId) {
+ private static void addToMap(Class<?> clazz, Byte byteId) {
CLASS_ID_MAP.put(clazz, byteId);
ID_CLASS_MAP.put(byteId, clazz);
}
@@ -338,7 +338,7 @@ public class MapWritable implements Writ
// read class-id map
fIdCount = in.readByte();
byte id;
- Class clazz;
+ Class<?> clazz;
for (int i = 0; i < fIdCount; i++) {
try {
id = in.readByte();
@@ -393,7 +393,7 @@ public class MapWritable implements Writ
}
}
- private byte addIdEntry(byte id, Class clazz) {
+ private byte addIdEntry(byte id, Class<?> clazz) {
if (fIdFirst == null) {
fIdFirst = fIdLast = new ClassIdEntry(id, clazz);
} else {
@@ -402,7 +402,7 @@ public class MapWritable implements Writ
return id;
}
- private byte getClassId(Class clazz) {
+ private byte getClassId(Class<?> clazz) {
Byte classId = CLASS_ID_MAP.get(clazz);
if (classId != null) {
return classId.byteValue();
@@ -438,8 +438,8 @@ public class MapWritable implements Writ
last = entry;
entry = entry.fNextEntry;
}
- Class keyClass = getClass(keyId);
- Class valueClass = getClass(valueId);
+ Class<?> keyClass = getClass(keyId);
+ Class<?> valueClass = getClass(valueId);
try {
return new KeyValueEntry((Writable) keyClass.newInstance(),
(Writable) valueClass.newInstance());
@@ -449,8 +449,8 @@ public class MapWritable implements Writ
}
- private Class getClass(final byte id) throws IOException {
- Class clazz = ID_CLASS_MAP.get(new Byte(id));
+ private Class<?> getClass(final byte id) throws IOException {
+ Class<?> clazz = ID_CLASS_MAP.get(new Byte(id));
if (clazz == null) {
ClassIdEntry entry = fIdFirst;
while (entry != null) {
@@ -502,14 +502,14 @@ public class MapWritable implements Writ
/** container for Id class tuples */
private class ClassIdEntry {
- public ClassIdEntry(byte id, Class clazz) {
+ public ClassIdEntry(byte id, Class<?> clazz) {
fId = id;
fclazz = clazz;
}
private byte fId;
- private Class fclazz;
+ private Class<?> fclazz;
private ClassIdEntry fNextIdEntry;
}
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/NutchWritable.java Tue May 21 01:19:26 2013
@@ -19,12 +19,13 @@ package org.apache.nutch.crawl;
import org.apache.hadoop.io.Writable;
import org.apache.nutch.util.GenericWritableConfigurable;
+@SuppressWarnings("unchecked")
public class NutchWritable extends GenericWritableConfigurable {
private static Class<? extends Writable>[] CLASSES = null;
static {
- CLASSES = new Class[] {
+ CLASSES = (Class<? extends Writable>[]) new Class<?>[] {
org.apache.hadoop.io.NullWritable.class,
org.apache.hadoop.io.BooleanWritable.class,
org.apache.hadoop.io.LongWritable.class,
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/SignatureComparator.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/SignatureComparator.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/SignatureComparator.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/SignatureComparator.java Tue May 21 01:19:26 2013
@@ -19,7 +19,7 @@ package org.apache.nutch.crawl;
import java.util.Comparator;
-public class SignatureComparator implements Comparator {
+public class SignatureComparator implements Comparator<Object> {
public int compare(Object o1, Object o2) {
return _compare(o1, o2);
}
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java Tue May 21 01:19:26 2013
@@ -47,7 +47,7 @@ public class SignatureFactory {
if (LOG.isInfoEnabled()) {
LOG.info("Using Signature impl: " + clazz);
}
- Class implClass = Class.forName(clazz);
+ Class<?> implClass = Class.forName(clazz);
impl = (Signature)implClass.newInstance();
impl.setConf(conf);
objectCache.setObject(clazz, impl);
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java Tue May 21 01:19:26 2013
@@ -45,7 +45,7 @@ import org.apache.nutch.util.*;
/** The fetcher. Most of the work is done by plugins. */
-public class OldFetcher extends Configured implements Tool, MapRunnable<WritableComparable, Writable, Text, NutchWritable> {
+public class OldFetcher extends Configured implements Tool, MapRunnable<WritableComparable<?>, Writable, Text, NutchWritable> {
public static final Logger LOG = LoggerFactory.getLogger(OldFetcher.class);
@@ -55,12 +55,11 @@ public class OldFetcher extends Configur
public static final String PROTOCOL_REDIR = "protocol";
- public static class InputFormat extends SequenceFileInputFormat<WritableComparable, Writable> {
+ public static class InputFormat extends SequenceFileInputFormat<WritableComparable<?>, Writable> {
/** Don't split inputs, to keep things polite. */
public InputSplit[] getSplits(JobConf job, int nSplits)
throws IOException {
FileStatus[] files = listStatus(job);
- FileSystem fs = FileSystem.get(job);
InputSplit[] splits = new InputSplit[files.length];
for (int i = 0; i < files.length; i++) {
FileStatus cur = files[i];
@@ -71,7 +70,7 @@ public class OldFetcher extends Configur
}
}
- private RecordReader<WritableComparable, Writable> input;
+ private RecordReader<WritableComparable<?>, Writable> input;
private OutputCollector<Text, NutchWritable> output;
private Reporter reporter;
@@ -458,7 +457,7 @@ public class OldFetcher extends Configur
return conf.getBoolean("fetcher.store.content", true);
}
- public void run(RecordReader<WritableComparable, Writable> input, OutputCollector<Text, NutchWritable> output,
+ public void run(RecordReader<WritableComparable<?>, Writable> input, OutputCollector<Text, NutchWritable> output,
Reporter reporter) throws IOException {
this.input = input;
Modified: nutch/trunk/src/java/org/apache/nutch/indexer/NutchField.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/NutchField.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/NutchField.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/NutchField.java Tue May 21 01:19:26 2013
@@ -28,8 +28,8 @@ import java.util.List;
import org.apache.hadoop.io.*;
/**
- * This class represents a multi-valued field with a weight. Values are arbitrary
- * objects.
+ * This class represents a multi-valued field with a weight.
+ * Values are arbitrary objects.
*/
public class NutchField implements Writable {
private float weight;
@@ -44,7 +44,7 @@ public class NutchField implements Writa
public NutchField(Object value, float weight) {
this.weight = weight;
if (value instanceof Collection) {
- values.addAll((Collection<Object>)value);
+ values.addAll((Collection<?>)value);
} else {
values.add(value);
}
Modified: nutch/trunk/src/java/org/apache/nutch/metadata/HttpHeaders.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/metadata/HttpHeaders.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/metadata/HttpHeaders.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/metadata/HttpHeaders.java Tue May 21 01:19:26 2013
@@ -23,9 +23,6 @@ import org.apache.hadoop.io.Text;
*
* @see <a href="http://rfc-ref.org/RFC-TEXTS/2616/">Hypertext Transfer
* Protocol -- HTTP/1.1 (RFC 2616)</a>
- *
- * @author Chris Mattmann
- * @author Jérôme Charron
*/
public interface HttpHeaders {
Modified: nutch/trunk/src/java/org/apache/nutch/metadata/Metadata.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/metadata/Metadata.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/metadata/Metadata.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/metadata/Metadata.java Tue May 21 01:19:26 2013
@@ -27,13 +27,8 @@ import java.util.Properties;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-
/**
* A multi-valued metadata container.
- *
- * @author Chris Mattmann
- * @author Jérôme Charron
- *
*/
public class Metadata implements Writable, CreativeCommons,
DublinCore, HttpHeaders, Nutch, Feed {
@@ -128,7 +123,7 @@ DublinCore, HttpHeaders, Nutch, Feed {
* @param properties properties to copy from
*/
public void setAll(Properties properties) {
- Enumeration names = properties.propertyNames();
+ Enumeration<?> names = properties.propertyNames();
while (names.hasMoreElements()) {
String name = (String) names.nextElement();
metadata.put(name, new String[]{properties.getProperty(name)});
Modified: nutch/trunk/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java Tue May 21 01:19:26 2013
@@ -52,9 +52,9 @@ public class SpellCheckedMetadata extend
// Uses following array to fill the metanames index and the
// metanames list.
- Class[] spellthese = {HttpHeaders.class};
+ Class<?>[] spellthese = {HttpHeaders.class};
- for (Class spellCheckedNames : spellthese) {
+ for (Class<?> spellCheckedNames : spellthese) {
for (Field field : spellCheckedNames.getFields()) {
int mods = field.getModifiers();
if (Modifier.isFinal(mods) && Modifier.isPublic(mods)
Modified: nutch/trunk/src/java/org/apache/nutch/net/URLNormalizers.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/net/URLNormalizers.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/net/URLNormalizers.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/net/URLNormalizers.java Tue May 21 01:19:26 2013
@@ -101,7 +101,7 @@ public final class URLNormalizers {
public static final Logger LOG = LoggerFactory.getLogger(URLNormalizers.class);
/* Empty extension list for caching purposes. */
- private final List<Extension> EMPTY_EXTENSION_LIST = Collections.EMPTY_LIST;
+ private final List<Extension> EMPTY_EXTENSION_LIST = Collections.<Extension>emptyList();
private final URLNormalizer[] EMPTY_NORMALIZERS = new URLNormalizer[0];
@@ -194,6 +194,7 @@ public final class URLNormalizers {
* empty list.
* @throws PluginRuntimeException
*/
+ @SuppressWarnings("unchecked")
private List<Extension> getExtensions(String scope) {
ObjectCache objectCache = ObjectCache.get(conf);
List<Extension> extensions =
Modified: nutch/trunk/src/java/org/apache/nutch/parse/HTMLMetaTags.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/HTMLMetaTags.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/HTMLMetaTags.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/HTMLMetaTags.java Tue May 21 01:19:26 2013
@@ -188,7 +188,7 @@ public class HTMLMetaTags {
+ ", refreshHref=" + refreshHref + "\n"
);
sb.append(" * general tags:\n");
- Iterator it = generalTags.keySet().iterator();
+ Iterator<Object> it = generalTags.keySet().iterator();
while (it.hasNext()) {
String key = (String)it.next();
sb.append(" - " + key + "\t=\t" + generalTags.get(key) + "\n");
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java Tue May 21 01:19:26 2013
@@ -42,7 +42,7 @@ import java.util.Map.Entry;
/* Parse content in a segment. */
public class ParseSegment extends Configured implements Tool,
- Mapper<WritableComparable, Content, Text, ParseImpl>,
+ Mapper<WritableComparable<?>, Content, Text, ParseImpl>,
Reducer<Text, Writable, Text, Writable> {
public static final Logger LOG = LoggerFactory.getLogger(ParseSegment.class);
@@ -71,7 +71,7 @@ public class ParseSegment extends Config
private Text newKey = new Text();
- public void map(WritableComparable key, Content content,
+ public void map(WritableComparable<?> key, Content content,
OutputCollector<Text, ParseImpl> output, Reporter reporter)
throws IOException {
// convert on the fly from old UTF8 keys
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java Tue May 21 01:19:26 2013
@@ -48,7 +48,7 @@ public final class ParserFactory {
public static final String DEFAULT_PLUGIN = "*";
/** Empty extension list for caching purposes. */
- private final List EMPTY_EXTENSION_LIST = Collections.EMPTY_LIST;
+ private final List<Extension> EMPTY_EXTENSION_LIST = Collections.<Extension>emptyList();
private Configuration conf;
private ExtensionPoint extensionPoint;
@@ -57,9 +57,9 @@ public final class ParserFactory {
public ParserFactory(Configuration conf) {
this.conf = conf;
ObjectCache objectCache = ObjectCache.get(conf);
- this.extensionPoint = PluginRepository.get(conf).getExtensionPoint(
- Parser.X_POINT_ID);
+ this.extensionPoint = PluginRepository.get(conf).getExtensionPoint(Parser.X_POINT_ID);
this.parsePluginList = (ParsePluginList)objectCache.getObject(ParsePluginList.class.getName());
+
if (this.parsePluginList == null) {
this.parsePluginList = new ParsePluginsReader().parse(conf);
objectCache.setObject(ParsePluginList.class.getName(), this.parsePluginList);
@@ -121,8 +121,8 @@ public final class ParserFactory {
}
parsers = new Vector<Parser>(parserExts.size());
- for (Iterator i=parserExts.iterator(); i.hasNext(); ){
- Extension ext = (Extension) i.next();
+ for (Iterator<Extension> i = parserExts.iterator(); i.hasNext(); ){
+ Extension ext = i.next();
Parser p = null;
try {
//check to see if we've cached this parser instance yet
@@ -212,6 +212,7 @@ public final class ParserFactory {
* @return a list of extensions to be used for this contentType.
* If none, returns <code>null</code>.
*/
+ @SuppressWarnings("unchecked")
protected List<Extension> getExtensions(String contentType) {
ObjectCache objectCache = ObjectCache.get(conf);
@@ -411,5 +412,4 @@ public final class ParserFactory {
private Extension getExtensionFromAlias(Extension[] list, String id) {
return getExtension(list, parsePluginList.getAliases().get(id));
}
-
}
Modified: nutch/trunk/src/java/org/apache/nutch/plugin/Extension.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/plugin/Extension.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/plugin/Extension.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/plugin/Extension.java Tue May 21 01:19:26 2013
@@ -25,8 +25,6 @@ import org.apache.hadoop.conf.Configurab
* An <code>Extension</code> is a kind of listener descriptor that will be
* installed on a concrete <code>ExtensionPoint</code> that acts as kind of
* Publisher.
- *
- * @author joa23
*/
public class Extension {
private PluginDescriptor fDescriptor;
@@ -153,7 +151,7 @@ public class Extension {
synchronized (getId()) {
try {
PluginClassLoader loader = fDescriptor.getClassLoader();
- Class extensionClazz = loader.loadClass(getClazz());
+ Class<?> extensionClazz = loader.loadClass(getClazz());
// lazy loading of Plugin in case there is no instance of the plugin
// already.
this.pluginRepository.getPluginInstance(getDescriptor());
Modified: nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java Tue May 21 01:19:26 2013
@@ -37,8 +37,6 @@ import org.apache.hadoop.conf.Configurat
* <code>ExtensionPoint</code> and <code>Extension</code>. To provide
* access to the meta data of a plugin via a descriptor allow a lazy loading
* mechanism.
- *
- * @author joa23
*/
public class PluginDescriptor {
private String fPluginPath;
@@ -47,7 +45,7 @@ public class PluginDescriptor {
private String fVersion;
private String fName;
private String fProviderName;
- private HashMap fMessages = new HashMap();
+ private HashMap<String, ResourceBundle> fMessages = new HashMap<String, ResourceBundle>();
private ArrayList<ExtensionPoint> fExtensionPoints = new ArrayList<ExtensionPoint>();
private ArrayList<String> fDependencies = new ArrayList<String>();
private ArrayList<URL> fExportedLibs = new ArrayList<URL>();
@@ -338,8 +336,7 @@ public class PluginDescriptor {
public String getResourceString(String pKey, Locale pLocale)
throws IOException {
if (fMessages.containsKey(pLocale.toString())) {
- ResourceBundle bundle = (ResourceBundle) fMessages
- .get(pLocale.toString());
+ ResourceBundle bundle = fMessages.get(pLocale.toString());
try {
return bundle.getString(pKey);
} catch (MissingResourceException e) {
Modified: nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java Tue May 21 01:19:26 2013
@@ -39,8 +39,6 @@ import org.apache.nutch.util.NutchConfig
* descriptor represents all meta information about a plugin. So a plugin
* instance will be created later when it is required, this allow lazy plugin
* loading.
- *
- * @author joa23
*/
public class PluginRepository {
private static final WeakHashMap<String, PluginRepository> CACHE = new WeakHashMap<String, PluginRepository>();
@@ -267,8 +265,8 @@ public class PluginRepository {
// Suggested by Stefan Groschupf <sg@media-style.com>
synchronized (pDescriptor) {
PluginClassLoader loader = pDescriptor.getClassLoader();
- Class pluginClass = loader.loadClass(pDescriptor.getPluginClass());
- Constructor constructor = pluginClass.getConstructor(new Class[] {
+ Class<?> pluginClass = loader.loadClass(pDescriptor.getPluginClass());
+ Constructor<?> constructor = pluginClass.getConstructor(new Class<?>[] {
PluginDescriptor.class, Configuration.class });
Plugin plugin = (Plugin) constructor.newInstance(new Object[] {
pDescriptor, this.conf });
@@ -400,7 +398,7 @@ public class PluginRepository {
}
ClassLoader cl = d.getClassLoader();
// args[1] - class name
- Class clazz = null;
+ Class<?> clazz = null;
try {
clazz = Class.forName(args[1], true, cl);
} catch (Exception e) {
@@ -410,7 +408,7 @@ public class PluginRepository {
}
Method m = null;
try {
- m = clazz.getMethod("main", new Class[] { args.getClass() });
+ m = clazz.getMethod("main", new Class<?>[] { args.getClass() });
} catch (Exception e) {
System.err.println("Could not find the 'main(String[])' method in class "
+ args[1] + ": " + e.getMessage());
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java Tue May 21 01:19:26 2013
@@ -433,12 +433,17 @@ public class LinkDumper
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option webGraphDbOpts = OptionBuilder.withArgName("webgraphdb").hasArg()
- .withDescription("the web graph database to use").create("webgraphdb");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the web graph database to use");
+ Option webGraphDbOpts = OptionBuilder.create("webgraphdb");
options.addOption(webGraphDbOpts);
+
CommandLineParser parser = new GnuParser();
try {
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java Tue May 21 01:19:26 2013
@@ -287,12 +287,10 @@ public class LinkRank
implements Mapper<Text, Node, Text, LongWritable>,
Reducer<Text, LongWritable, Text, LongWritable> {
- private JobConf conf;
private static Text numNodes = new Text(NUM_NODES);
private static LongWritable one = new LongWritable(1L);
public void configure(JobConf conf) {
- this.conf = conf;
}
/**
@@ -678,11 +676,15 @@ public class LinkRank
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option webgraphOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
- "the web graph db to use").create("webgraphdb");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the web graph db to use");
+ Option webgraphOpts = OptionBuilder.create("webgraphdb");
options.addOption(webgraphOpts);
CommandLineParser parser = new GnuParser();
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LoopReader.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LoopReader.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LoopReader.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/LoopReader.java Tue May 21 01:19:26 2013
@@ -45,9 +45,7 @@ public class LoopReader extends Configur
private FileSystem fs;
private MapFile.Reader[] loopReaders;
- public LoopReader() {
-
- }
+ public LoopReader() { }
public LoopReader(Configuration conf) {
super(conf);
@@ -94,14 +92,21 @@ public class LoopReader extends Configur
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option webGraphOpts = OptionBuilder.withArgName("webgraphdb").hasArg()
- .withDescription("the webgraphdb to use").create("webgraphdb");
- Option urlOpts = OptionBuilder.withArgName("url").hasOptionalArg()
- .withDescription("the url to dump").create("url");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the webgraphdb to use");
+ Option webGraphOpts = OptionBuilder.create("webgraphdb");
options.addOption(webGraphOpts);
+
+ OptionBuilder.withArgName("url");
+ OptionBuilder.hasOptionalArg();
+ OptionBuilder.withDescription("the url to dump");
+ Option urlOpts = OptionBuilder.create("url");
options.addOption(urlOpts);
CommandLineParser parser = new GnuParser();
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/Loops.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/Loops.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/Loops.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/Loops.java Tue May 21 01:19:26 2013
@@ -583,11 +583,15 @@ public class Loops
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option webGraphDbOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
- "the web graph database to use").create("webgraphdb");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the web graph database to use");
+ Option webGraphDbOpts = OptionBuilder.create("webgraphdb");
options.addOption(webGraphDbOpts);
CommandLineParser parser = new GnuParser();
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeDumper.java Tue May 21 01:19:26 2013
@@ -343,36 +343,57 @@ public class NodeDumper
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option webGraphDbOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
- "the web graph database to use").create("webgraphdb");
- Option inlinkOpts = OptionBuilder.withArgName("inlinks").withDescription(
- "show highest inlinks").create("inlinks");
- Option outlinkOpts = OptionBuilder.withArgName("outlinks").withDescription(
- "show highest outlinks").create("outlinks");
- Option scoreOpts = OptionBuilder.withArgName("scores").withDescription(
- "show highest scores").create("scores");
- Option topNOpts = OptionBuilder.withArgName("topn").hasOptionalArg().withDescription(
- "show topN scores").create("topn");
- Option outputOpts = OptionBuilder.withArgName("output").hasArg().withDescription(
- "the output directory to use").create("output");
- Option effOpts = OptionBuilder.withArgName("asEff").withDescription(
- "Solr ExternalFileField compatible output format").create("asEff");
- Option groupOpts = OptionBuilder.hasArgs(2).withDescription(
- "group <host|domain> <sum|max>").create("group");
- Option sequenceFileOpts = OptionBuilder.withArgName("asSequenceFile").withDescription(
- "whether to output as a sequencefile").create("asSequenceFile");
-
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the web graph database to use");
+ Option webGraphDbOpts = OptionBuilder.create("webgraphdb");
options.addOption(webGraphDbOpts);
+
+ OptionBuilder.withArgName("inlinks");
+ OptionBuilder.withDescription("show highest inlinks");
+ Option inlinkOpts = OptionBuilder.create("inlinks");
options.addOption(inlinkOpts);
+
+ OptionBuilder.withArgName("outlinks");
+ OptionBuilder.withDescription("show highest outlinks");
+ Option outlinkOpts = OptionBuilder.create("outlinks");
options.addOption(outlinkOpts);
+
+ OptionBuilder.withArgName("scores");
+ OptionBuilder.withDescription("show highest scores");
+ Option scoreOpts = OptionBuilder.create("scores");
options.addOption(scoreOpts);
+
+ OptionBuilder.withArgName("topn");
+ OptionBuilder.hasOptionalArg();
+ OptionBuilder.withDescription("show topN scores");
+ Option topNOpts = OptionBuilder.create("topn");
options.addOption(topNOpts);
+
+ OptionBuilder.withArgName("output");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the output directory to use");
+ Option outputOpts = OptionBuilder.create("output");
options.addOption(outputOpts);
+
+ OptionBuilder.withArgName("asEff");
+ OptionBuilder.withDescription("Solr ExternalFileField compatible output format");
+ Option effOpts = OptionBuilder.create("asEff");
options.addOption(effOpts);
+
+ OptionBuilder.hasArgs(2);
+ OptionBuilder.withDescription("group <host|domain> <sum|max>");
+ Option groupOpts = OptionBuilder.create("group");
options.addOption(groupOpts);
+
+ OptionBuilder.withArgName("asSequenceFile");
+ OptionBuilder.withDescription("whether to output as a sequencefile");
+ Option sequenceFileOpts = OptionBuilder.create("asSequenceFile");
options.addOption(sequenceFileOpts);
CommandLineParser parser = new GnuParser();
@@ -388,7 +409,6 @@ public class NodeDumper
String webGraphDb = line.getOptionValue("webgraphdb");
boolean inlinks = line.hasOption("inlinks");
boolean outlinks = line.hasOption("outlinks");
- boolean scores = line.hasOption("scores");
long topN = (line.hasOption("topn")
? Long.parseLong(line.getOptionValue("topn")) : Long.MAX_VALUE);
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/NodeReader.java Tue May 21 01:19:26 2013
@@ -90,14 +90,21 @@ public class NodeReader extends Configur
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option webGraphOpts = OptionBuilder.withArgName("webgraphdb").hasArg()
- .withDescription("the webgraphdb to use").create("webgraphdb");
- Option urlOpts = OptionBuilder.withArgName("url").hasOptionalArg()
- .withDescription("the url to dump").create("url");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the webgraphdb to use");
+ Option webGraphOpts = OptionBuilder.create("webgraphdb");
options.addOption(webGraphOpts);
+
+ OptionBuilder.withArgName("url");
+ OptionBuilder.hasOptionalArg();
+ OptionBuilder.withDescription("the url to dump");
+ Option urlOpts = OptionBuilder.create("url");
options.addOption(urlOpts);
CommandLineParser parser = new GnuParser();
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/ScoreUpdater.java Tue May 21 01:19:26 2013
@@ -217,14 +217,21 @@ public class ScoreUpdater
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option crawlDbOpts = OptionBuilder.withArgName("crawldb").hasArg().withDescription(
- "the crawldb to use").create("crawldb");
- Option webGraphOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
- "the webgraphdb to use").create("webgraphdb");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("crawldb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the crawldb to use");
+ Option crawlDbOpts = OptionBuilder.create("crawldb");
options.addOption(crawlDbOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the webgraphdb to use");
+ Option webGraphOpts = OptionBuilder.create("webgraphdb");
options.addOption(webGraphOpts);
CommandLineParser parser = new GnuParser();
Modified: nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/scoring/webgraph/WebGraph.java Tue May 21 01:19:26 2013
@@ -405,28 +405,13 @@ public class WebGraph
extends Configured
implements Mapper<Text, LinkDatum, Text, LinkDatum> {
- private JobConf conf;
private long timestamp;
/**
- * Default constructor.
- */
- public InlinkDb() {
- }
-
- /**
- * Configurable constructor.
- */
- public InlinkDb(Configuration conf) {
- setConf(conf);
- }
-
- /**
* Configures job. Sets timestamp for all Inlink LinkDatum objects to the
* current system time.
*/
public void configure(JobConf conf) {
- this.conf = conf;
timestamp = System.currentTimeMillis();
}
@@ -461,30 +446,12 @@ public class WebGraph
extends Configured
implements Reducer<Text, LinkDatum, Text, Node> {
- private JobConf conf;
-
- /**
- * Default constructor.
- */
- public NodeDb() {
- }
-
- /**
- * Configurable constructor.
- */
- public NodeDb(Configuration conf) {
- setConf(conf);
- }
-
/**
* Configures job.
*/
- public void configure(JobConf conf) {
- this.conf = conf;
- }
+ public void configure(JobConf conf) { }
- public void close() {
- }
+ public void close() { }
/**
* Counts the number of inlinks and outlinks for each url and sets a default
@@ -731,23 +698,37 @@ public class WebGraph
throws Exception {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option webGraphDbOpts = OptionBuilder.withArgName("webgraphdb").hasArg().withDescription(
- "the web graph database to use").create("webgraphdb");
- Option segOpts = OptionBuilder.withArgName("segment").hasArgs().withDescription(
- "the segment(s) to use").create("segment");
- Option segDirOpts = OptionBuilder.withArgName("segmentDir").hasArgs().withDescription(
- "the segment directory to use").create("segmentDir");
- Option normalizeOpts = OptionBuilder.withArgName("normalize").withDescription(
- "whether to use URLNormalizers on the URL's in the segment").create("normalize");
- Option filterOpts = OptionBuilder.withArgName("filter").withDescription(
- "whether to use URLFilters on the URL's in the segment").create("filter");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("webgraphdb");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the web graph database to use");
+ Option webGraphDbOpts = OptionBuilder.create("webgraphdb");
options.addOption(webGraphDbOpts);
+
+ OptionBuilder.withArgName("segment");
+ OptionBuilder.hasArgs();
+ OptionBuilder.withDescription("the segment(s) to use");
+ Option segOpts = OptionBuilder.create("segment");
options.addOption(segOpts);
+
+ OptionBuilder.withArgName("segmentDir");
+ OptionBuilder.hasArgs();
+ OptionBuilder.withDescription("the segment directory to use");
+ Option segDirOpts = OptionBuilder.create("segmentDir");
options.addOption(segDirOpts);
+
+ OptionBuilder.withArgName("normalize");
+ OptionBuilder.withDescription("whether to use URLNormalizers on the URL's in the segment");
+ Option normalizeOpts = OptionBuilder.create("normalize");
options.addOption(normalizeOpts);
+
+ OptionBuilder.withArgName("filter");
+ OptionBuilder.withDescription("whether to use URLFilters on the URL's in the segment");
+ Option filterOpts = OptionBuilder.create("filter");
options.addOption(filterOpts);
CommandLineParser parser = new GnuParser();
Modified: nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilter.java Tue May 21 01:19:26 2013
@@ -18,7 +18,7 @@ package org.apache.nutch.segment;
import java.util.Collection;
-import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.parse.ParseText;
@@ -41,7 +41,7 @@ public interface SegmentMergeFilter {
* @return <tt>true</tt> values for this <tt>key</tt> (URL) should be merged
* into the new segment.
*/
- public boolean filter(WritableComparable key, CrawlDatum generateData,
+ public boolean filter(Text key, CrawlDatum generateData,
CrawlDatum fetchData, CrawlDatum sigData, Content content,
ParseData parseData, ParseText parseText, Collection<CrawlDatum> linked);
}
Modified: nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentMergeFilters.java Tue May 21 01:19:26 2013
@@ -21,7 +21,7 @@ import java.util.Collection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.URLFilter;
import org.apache.nutch.parse.ParseData;
@@ -65,16 +65,14 @@ public class SegmentMergeFilters {
* @return <tt>true</tt> values for this <tt>key</tt> (URL) should be merged
* into the new segment.
*/
- public boolean filter(WritableComparable key, CrawlDatum generateData,
+ public boolean filter(Text key, CrawlDatum generateData,
CrawlDatum fetchData, CrawlDatum sigData, Content content,
ParseData parseData, ParseText parseText, Collection<CrawlDatum> linked) {
for (SegmentMergeFilter filter : filters) {
if (!filter.filter(key, generateData, fetchData, sigData, content,
parseData, parseText, linked)) {
if (LOG.isTraceEnabled())
- LOG
- .trace("Key " + key + " dropped by "
- + filter.getClass().getName());
+ LOG.trace("Key " + key + " dropped by " + filter.getClass().getName());
return false;
}
}
Modified: nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java Tue May 21 01:19:26 2013
@@ -16,6 +16,7 @@
*/
package org.apache.nutch.segment;
+import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -207,7 +208,7 @@ public class SegmentMerger extends Confi
MapFile.Writer pt_out = null;
SequenceFile.Writer g_out = null;
SequenceFile.Writer p_out = null;
- HashMap sliceWriters = new HashMap();
+ HashMap<String, Closeable> sliceWriters = new HashMap<String, Closeable>();
String segmentName = job.get("segment.merger.segmentName");
public void write(Text key, MetaWrapper wrapper) throws IOException {
@@ -288,7 +289,7 @@ public class SegmentMerger extends Confi
}
public void close(Reporter reporter) throws IOException {
- Iterator<Object> it = sliceWriters.values().iterator();
+ Iterator<Closeable> it = sliceWriters.values().iterator();
while (it.hasNext()) {
Object o = it.next();
if (o instanceof SequenceFile.Writer) {
Modified: nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java Tue May 21 01:19:26 2013
@@ -80,10 +80,10 @@ public class SegmentReader extends Confi
private FileSystem fs;
public static class InputCompatMapper extends MapReduceBase implements
- Mapper<WritableComparable, Writable, Text, NutchWritable> {
+ Mapper<WritableComparable<?>, Writable, Text, NutchWritable> {
private Text newKey = new Text();
- public void map(WritableComparable key, Writable value,
+ public void map(WritableComparable<?> key, Writable value,
OutputCollector<Text, NutchWritable> collector, Reporter reporter) throws IOException {
// convert on the fly from old formats with UTF8 keys.
// UTF8 deprecated and replaced by Text.
@@ -98,8 +98,8 @@ public class SegmentReader extends Confi
/** Implements a text output format */
public static class TextOutputFormat extends
- FileOutputFormat<WritableComparable, Writable> {
- public RecordWriter<WritableComparable, Writable> getRecordWriter(
+ FileOutputFormat<WritableComparable<?>, Writable> {
+ public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(
final FileSystem fs, JobConf job,
String name, final Progressable progress) throws IOException {
@@ -109,8 +109,8 @@ public class SegmentReader extends Confi
if (fs.exists(segmentDumpFile)) fs.delete(segmentDumpFile, true);
final PrintStream printStream = new PrintStream(fs.create(segmentDumpFile));
- return new RecordWriter<WritableComparable, Writable>() {
- public synchronized void write(WritableComparable key, Writable value) throws IOException {
+ return new RecordWriter<WritableComparable<?>, Writable>() {
+ public synchronized void write(WritableComparable<?> key, Writable value) throws IOException {
printStream.println(value);
}
@@ -379,8 +379,8 @@ public class SegmentReader extends Confi
private List<Writable> getMapRecords(Path dir, Text key) throws Exception {
MapFile.Reader[] readers = MapFileOutputFormat.getReaders(fs, dir, getConf());
ArrayList<Writable> res = new ArrayList<Writable>();
- Class keyClass = readers[0].getKeyClass();
- Class valueClass = readers[0].getValueClass();
+ Class<?> keyClass = readers[0].getKeyClass();
+ Class<?> valueClass = readers[0].getValueClass();
if (!keyClass.getName().equals("org.apache.hadoop.io.Text"))
throw new IOException("Incompatible key (" + keyClass.getName() + ")");
Writable value = (Writable)valueClass.newInstance();
@@ -403,8 +403,8 @@ public class SegmentReader extends Confi
private List<Writable> getSeqRecords(Path dir, Text key) throws Exception {
SequenceFile.Reader[] readers = SequenceFileOutputFormat.getReaders(getConf(), dir);
ArrayList<Writable> res = new ArrayList<Writable>();
- Class keyClass = readers[0].getKeyClass();
- Class valueClass = readers[0].getValueClass();
+ Class<?> keyClass = readers[0].getKeyClass();
+ Class<?> valueClass = readers[0].getValueClass();
if (!keyClass.getName().equals("org.apache.hadoop.io.Text"))
throw new IOException("Incompatible key (" + keyClass.getName() + ")");
Writable aKey = (Writable)keyClass.newInstance();
Modified: nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/tools/FreeGenerator.java Tue May 21 01:19:26 2013
@@ -57,8 +57,6 @@ import org.apache.nutch.util.TimingUtil;
* This tool generates fetchlists (segments to be fetched) from plain text
* files containing one URL per line. It's useful when arbitrary URL-s need to
* be fetched without adding them first to the CrawlDb, or during testing.
- *
- * @author Andrzej Bialecki
*/
public class FreeGenerator extends Configured implements Tool {
private static final Logger LOG = LoggerFactory.getLogger(FreeGenerator.class);
@@ -67,7 +65,7 @@ public class FreeGenerator extends Confi
private static final String NORMALIZE_KEY = "free.generator.normalize";
public static class FG extends MapReduceBase
- implements Mapper<WritableComparable, Text, Text, Generator.SelectorEntry>,
+ implements Mapper<WritableComparable<?>, Text, Text, Generator.SelectorEntry>,
Reducer<Text, Generator.SelectorEntry, Text, CrawlDatum> {
private URLNormalizers normalizers = null;
private URLFilters filters = null;
@@ -89,7 +87,7 @@ public class FreeGenerator extends Confi
Generator.SelectorEntry entry = new Generator.SelectorEntry();
- public void map(WritableComparable key, Text value, OutputCollector<Text,
+ public void map(WritableComparable<?> key, Text value, OutputCollector<Text,
Generator.SelectorEntry> output, Reporter reporter) throws IOException {
// value is a line of text
String urlString = value.toString();
Modified: nutch/trunk/src/java/org/apache/nutch/tools/ResolveUrls.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/tools/ResolveUrls.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/tools/ResolveUrls.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/tools/ResolveUrls.java Tue May 21 01:19:26 2013
@@ -77,7 +77,7 @@ public class ResolveUrls {
// get the address by name and if no error is thrown then it
// is resolved successfully
- InetAddress ia = InetAddress.getByName(host);
+ InetAddress.getByName(host);
LOG.info("Resolved: " + host);
numResolved.incrementAndGet();
}
@@ -161,19 +161,25 @@ public class ResolveUrls {
public static void main(String[] args) {
Options options = new Options();
- Option helpOpts = OptionBuilder.withArgName("help").withDescription(
- "show this help message").create("help");
- Option urlOpts = OptionBuilder.withArgName("urls").hasArg().withDescription(
- "the urls file to check").create("urls");
- Option numThreadOpts = OptionBuilder.withArgName("numThreads").hasArgs().withDescription(
- "the number of threads to use").create("numThreads");
+ OptionBuilder.withArgName("help");
+ OptionBuilder.withDescription("show this help message");
+ Option helpOpts = OptionBuilder.create("help");
options.addOption(helpOpts);
+
+ OptionBuilder.withArgName("urls");
+ OptionBuilder.hasArg();
+ OptionBuilder.withDescription("the urls file to check");
+ Option urlOpts = OptionBuilder.create("urls");
options.addOption(urlOpts);
+
+ OptionBuilder.withArgName("numThreads");
+ OptionBuilder.hasArgs();
+ OptionBuilder.withDescription("the number of threads to use");
+ Option numThreadOpts = OptionBuilder.create("numThreads");
options.addOption(numThreadOpts);
CommandLineParser parser = new GnuParser();
try {
-
// parse out common line arguments
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || !line.hasOption("urls")) {
@@ -196,5 +202,4 @@ public class ResolveUrls {
LOG.error("ResolveUrls: " + StringUtils.stringifyException(e));
}
}
-
}
Modified: nutch/trunk/src/java/org/apache/nutch/tools/proxy/SegmentHandler.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/tools/proxy/SegmentHandler.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/tools/proxy/SegmentHandler.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/tools/proxy/SegmentHandler.java Tue May 21 01:19:26 2013
@@ -42,8 +42,6 @@ import org.apache.hadoop.util.StringUtil
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.Nutch;
-import org.apache.nutch.parse.ParseData;
-import org.apache.nutch.parse.ParseText;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.ProtocolStatus;
import org.mortbay.jetty.Request;
@@ -88,9 +86,8 @@ public class SegmentHandler extends Abst
private static class Segment implements Closeable {
- private static final Partitioner PARTITIONER = new HashPartitioner();
+ private static final Partitioner<Text,Writable> PARTITIONER = new HashPartitioner<Text,Writable>();
- private FileSystem fs;
private Path segmentDir;
private Object cLock = new Object();
@@ -102,7 +99,6 @@ public class SegmentHandler extends Abst
private Configuration conf;
public Segment(FileSystem fs, Path segmentDir, Configuration conf) throws IOException {
- this.fs = fs;
this.segmentDir = segmentDir;
this.conf = conf;
}
Modified: nutch/trunk/src/java/org/apache/nutch/util/GenericWritableConfigurable.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/GenericWritableConfigurable.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/GenericWritableConfigurable.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/util/GenericWritableConfigurable.java Tue May 21 01:19:26 2013
@@ -41,7 +41,7 @@ public abstract class GenericWritableCon
@Override
public void readFields(DataInput in) throws IOException {
byte type = in.readByte();
- Class clazz = getTypes()[type];
+ Class<?> clazz = getTypes()[type];
try {
set((Writable) clazz.newInstance());
} catch (Exception e) {
Modified: nutch/trunk/src/java/org/apache/nutch/util/PrefixStringMatcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/PrefixStringMatcher.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/PrefixStringMatcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/util/PrefixStringMatcher.java Tue May 21 01:19:26 2013
@@ -45,11 +45,11 @@ public class PrefixStringMatcher extends
* @throws ClassCastException if any <code>Object</code>s in the
* collection are not <code>String</code>s
*/
- public PrefixStringMatcher(Collection prefixes) {
+ public PrefixStringMatcher(Collection<String> prefixes) {
super();
- Iterator iter= prefixes.iterator();
+ Iterator<String> iter= prefixes.iterator();
while (iter.hasNext())
- addPatternForward((String)iter.next());
+ addPatternForward(iter.next());
}
/**
Modified: nutch/trunk/src/java/org/apache/nutch/util/SuffixStringMatcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/SuffixStringMatcher.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/SuffixStringMatcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/util/SuffixStringMatcher.java Tue May 21 01:19:26 2013
@@ -41,11 +41,11 @@ public class SuffixStringMatcher extends
* <code>String</code>s with any suffix in the supplied
* <code>Collection</code>
*/
- public SuffixStringMatcher(Collection suffixes) {
+ public SuffixStringMatcher(Collection<String> suffixes) {
super();
- Iterator iter= suffixes.iterator();
+ Iterator<String> iter= suffixes.iterator();
while (iter.hasNext())
- addPatternBackward((String)iter.next());
+ addPatternBackward(iter.next());
}
/**
Modified: nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java (original)
+++ nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java Tue May 21 01:19:26 2013
@@ -22,7 +22,6 @@ import org.apache.nutch.parse.*;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.metadata.Metadata;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -217,11 +216,6 @@ public class CCParseFilter implements Ht
if (!CC_NS.equals(predicateElement.getNamespaceURI())) {
continue;
}
- String predicate = predicateElement.getLocalName();
-
- // object is rdf:resource from cc:xxx predicates
- String object =
- predicateElement.getAttributeNodeNS(RDF_NS, "resource").getValue();
// add object and predicate to metadata
// metadata.put(object, predicate);
@@ -234,22 +228,19 @@ public class CCParseFilter implements Ht
// get cc:Work nodes from rdf:RDF
NodeList works = rdf.getElementsByTagNameNS(CC_NS, "Work");
for (int i = 0; i < works.getLength(); i++) {
- Element l = (Element)works.item(i);
-
// get dc:type nodes from cc:Work
NodeList types = rdf.getElementsByTagNameNS(DC_NS, "type");
+
for (int j = 0; j < types.getLength(); j++) {
Element type = (Element)types.item(j);
- String workUri =
- type.getAttributeNodeNS(RDF_NS, "resource").getValue();
- this.workType = (String)WORK_TYPE_NAMES.get(workUri);
- break;
+ String workUri = type.getAttributeNodeNS(RDF_NS, "resource").getValue();
+ this.workType = WORK_TYPE_NAMES.get(workUri);
}
}
}
}
- private static final HashMap WORK_TYPE_NAMES = new HashMap();
+ private static final HashMap<String, String> WORK_TYPE_NAMES = new HashMap<String, String>();
static {
WORK_TYPE_NAMES.put("http://purl.org/dc/dcmitype/MovingImage", "video");
WORK_TYPE_NAMES.put("http://purl.org/dc/dcmitype/StillImage", "image");
Modified: nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java (original)
+++ nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java Tue May 21 01:19:26 2013
@@ -109,7 +109,7 @@ public abstract class RegexURLFilterBase
private static FilteredURL[] readURLFile(Reader reader) throws IOException {
BufferedReader in = new BufferedReader(reader);
- List list = new ArrayList();
+ List<FilteredURL> list = new ArrayList<FilteredURL>();
String line;
while((line=in.readLine()) != null) {
if (line.length() != 0) {
Modified: nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java (original)
+++ nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java Tue May 21 01:19:26 2013
@@ -42,15 +42,12 @@ import org.apache.nutch.util.StringUtil;
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
-
/**
* Adds microformat rel-tags of document if found.
*
* @see <a href="http://www.microformats.org/wiki/rel-tag">
* http://www.microformats.org/wiki/rel-tag</a>
- * @author Jérôme Charron
*/
public class RelTagParser implements HtmlParseFilter {
@@ -58,10 +55,8 @@ public class RelTagParser implements Htm
public final static String REL_TAG = "Rel-Tag";
-
private Configuration conf = null;
-
/**
* Scan the HTML document looking at possible rel-tags
*/
@@ -72,25 +67,25 @@ public class RelTagParser implements Htm
Parse parse = parseResult.get(content.getUrl());
// Trying to find the document's rel-tags
Parser parser = new Parser(doc);
- Set tags = parser.getRelTags();
- Iterator iter = tags.iterator();
+ Set<?> tags = parser.getRelTags();
+ Iterator<?> iter = tags.iterator();
Metadata metadata = parse.getData().getParseMeta();
- while (iter.hasNext()) {
+ while (iter.hasNext())
metadata.add(REL_TAG, (String) iter.next());
- }
+
return parseResult;
}
private static class Parser {
- Set tags = null;
+ Set<String> tags = null;
Parser(Node node) {
- tags = new TreeSet();
+ tags = new TreeSet<String>();
parse(node);
}
- Set getRelTags() {
+ Set<String> getRelTags() {
return tags;
}
@@ -120,9 +115,8 @@ public class RelTagParser implements Htm
// Recurse
NodeList children = node.getChildNodes();
- for (int i=0; children != null && i<children.getLength(); i++) {
+ for (int i=0; children != null && i<children.getLength(); i++)
parse(children.item(i));
- }
}
private final static String parseTag(String url) {
@@ -140,11 +134,6 @@ public class RelTagParser implements Htm
}
-
- /* ----------------------------- *
- * <implementation:Configurable> *
- * ----------------------------- */
-
public void setConf(Configuration conf) {
this.conf = conf;
}
@@ -152,9 +141,4 @@ public class RelTagParser implements Htm
public Configuration getConf() {
return this.conf;
}
-
- /* ------------------------------ *
- * </implementation:Configurable> *
- * ------------------------------ */
-
}
Modified: nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java (original)
+++ nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java Tue May 21 01:19:26 2013
@@ -21,14 +21,12 @@ import org.apache.nutch.protocol.Content
import org.apache.nutch.parse.ParseResult;
import org.apache.nutch.parse.ParseStatus;
import org.apache.nutch.parse.Parser;
-import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.parse.ParseImpl;
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.parse.OutlinkExtractor;
import org.apache.nutch.util.CommandRunner;
-import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.net.protocols.Response;
import org.apache.hadoop.conf.Configuration;
@@ -59,12 +57,10 @@ public class ExtParser implements Parser
static final int TIMEOUT_DEFAULT = 30; // in seconds
// handy map from String contentType to String[] {command, timeoutString, encoding}
- Hashtable TYPE_PARAMS_MAP = new Hashtable();
+ Hashtable<String, String[]> TYPE_PARAMS_MAP = new Hashtable<String, String[]>();
private Configuration conf;
- private boolean loaded = false;
-
public ExtParser () { }
public ParseResult getParse(Content content) {
Modified: nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMBuilder.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMBuilder.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMBuilder.java (original)
+++ nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/DOMBuilder.java Tue May 21 01:19:26 2013
@@ -58,7 +58,7 @@ public class DOMBuilder
public DocumentFragment m_docFrag = null;
/** Vector of element nodes */
- protected Stack m_elemStack = new Stack();
+ protected Stack<Element> m_elemStack = new Stack<Element>();
/**
* DOMBuilder instance constructor... it will add the DOM nodes
Modified: nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java?rev=1484634&r1=1484633&r2=1484634&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java (original)
+++ nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java Tue May 21 01:19:26 2013
@@ -42,7 +42,6 @@ import org.apache.nutch.parse.Parser;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
import org.apache.oro.text.regex.MatchResult;
import org.apache.oro.text.regex.Pattern;
import org.apache.oro.text.regex.PatternCompiler;
@@ -60,9 +59,6 @@ import org.w3c.dom.NodeList;
* This class is a heuristic link extractor for JavaScript files and
* code snippets. The general idea of a two-pass regex matching comes from
* Heritrix. Parts of the code come from OutlinkExtractor.java
- * by Stephan Strittmatter.
- *
- * @author Andrzej Bialecki <ab@getopt.org>
*/
public class JSParseFilter implements HtmlParseFilter, Parser {
public static final Logger LOG = LoggerFactory.getLogger(JSParseFilter.class);
@@ -77,12 +73,12 @@ public class JSParseFilter implements Ht
Parse parse = parseResult.get(content.getUrl());
String url = content.getBaseUrl();
- ArrayList outlinks = new ArrayList();
+ ArrayList<Outlink> outlinks = new ArrayList<Outlink>();
walk(doc, parse, metaTags, url, outlinks);
if (outlinks.size() > 0) {
Outlink[] old = parse.getData().getOutlinks();
String title = parse.getData().getTitle();
- List list = Arrays.asList(old);
+ List<Outlink> list = Arrays.asList(old);
outlinks.addAll(list);
ParseStatus status = parse.getData().getStatus();
String text = parse.getText();
@@ -97,14 +93,14 @@ public class JSParseFilter implements Ht
return parseResult;
}
- private void walk(Node n, Parse parse, HTMLMetaTags metaTags, String base, List outlinks) {
+ private void walk(Node n, Parse parse, HTMLMetaTags metaTags, String base, List<Outlink> outlinks) {
if (n instanceof Element) {
String name = n.getNodeName();
if (name.equalsIgnoreCase("script")) {
- String lang = null;
+ /* String lang = null;
Node lNode = n.getAttributes().getNamedItem("language");
if (lNode == null) lang = "javascript";
- else lang = lNode.getNodeValue();
+ else lang = lNode.getNodeValue(); */
StringBuffer script = new StringBuffer();
NodeList nn = n.getChildNodes();
if (nn.getLength() > 0) {
@@ -183,7 +179,7 @@ public class JSParseFilter implements Ht
*/
private Outlink[] getJSLinks(String plainText, String anchor, String base) {
- final List outlinks = new ArrayList();
+ final List<Outlink> outlinks = new ArrayList<Outlink>();
URL baseURL = null;
try {
@@ -265,7 +261,10 @@ public class JSParseFilter implements Ht
BufferedReader br = new BufferedReader(new InputStreamReader(in, "UTF-8"));
StringBuffer sb = new StringBuffer();
String line = null;
- while ((line = br.readLine()) != null) sb.append(line + "\n");
+ while ((line = br.readLine()) != null)
+ sb.append(line + "\n");
+ br.close();
+
JSParseFilter parseFilter = new JSParseFilter();
parseFilter.setConf(NutchConfiguration.create());
Outlink[] links = parseFilter.getJSLinks(sb.toString(), "", args[1]);
Author: tejasp
Date: Tue May 21 00:40:34 2013
New Revision: 1484628
URL: http://svn.apache.org/r1484628
Log:
NUTCH-1053 Parsing of RSS feeds fails
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/plugin/feed/ivy.xml
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1484628&r1=1484627&r2=1484628&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue May 21 00:40:34 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1053 Parsing of RSS feeds fails (tejasp)
+
* Added crawler-commons dependency in pom.xml (tejasp)
* NUTCH-956 solrindex issues: add field tld to Solr schema (Alexis via lewismc, snagel)
Modified: nutch/trunk/src/plugin/feed/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/feed/ivy.xml?rev=1484628&r1=1484627&r2=1484628&view=diff
==============================================================================
--- nutch/trunk/src/plugin/feed/ivy.xml (original)
+++ nutch/trunk/src/plugin/feed/ivy.xml Tue May 21 00:40:34 2013
@@ -37,6 +37,7 @@
<dependencies>
<dependency org="rome" name="rome" rev="0.9" conf="*->master"/>
+ <dependency org="org.jdom" name="jdom" rev="1.1" conf="*->master"/>
</dependencies>
</ivy-module>
Author: tejasp
Date: Tue May 21 00:40:16 2013
New Revision: 1484627
URL: http://svn.apache.org/r1484627
Log:
NUTCH-1053 Parsing of RSS feeds fails
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/plugin/feed/ivy.xml
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1484627&r1=1484626&r2=1484627&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Tue May 21 00:40:16 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1053 Parsing of RSS feeds fails (tejasp)
+
* NUTCH-1563 FetchSchedule#getFields is never used by GeneratorJob (Feng)
* NUTCH-1573 Upgrade to most recent JUnit 4.x to improve test flexibility (lewismc)
Modified: nutch/branches/2.x/src/plugin/feed/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/feed/ivy.xml?rev=1484627&r1=1484626&r2=1484627&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/feed/ivy.xml (original)
+++ nutch/branches/2.x/src/plugin/feed/ivy.xml Tue May 21 00:40:16 2013
@@ -37,6 +37,7 @@
<dependencies>
<dependency org="net.java.dev.rome" name="rome" rev="1.0.0" conf="*->master"/>
+ <dependency org="org.jdom" name="jdom" rev="1.1" conf="*->master"/>
</dependencies>
</ivy-module>
Author: fenglu
Date: Mon May 20 13:44:11 2013
New Revision: 1484482
URL: http://svn.apache.org/r1484482
Log:
NUTCH-1563 FetchSchedule#getFields is never used by GeneraterJob
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1484482&r1=1484481&r2=1484482&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Mon May 20 13:44:11 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1563 FetchSchedule#getFields is never used by GeneratorJob (Feng)
+
* NUTCH-1573 Upgrade to most recent JUnit 4.x to improve test flexibility (lewismc)
* Added crawler-commons dependency in pom.xml (tejasp)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java?rev=1484482&r1=1484481&r2=1484482&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java Mon May 20 13:44:11
2013
@@ -24,7 +24,9 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.Collection;
+import org.apache.hadoop.mapreduce.Job;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
@@ -152,6 +154,12 @@ public class GeneratorJob extends NutchT
setConf(conf);
}
+ public Collection<WebPage.Field> getFields(Job job) {
+ Collection<WebPage.Field> fields = new HashSet<WebPage.Field>(FIELDS);
+ fields.addAll(FetchScheduleFactory.getFetchSchedule(job.getConfiguration()).getFields());
+ return fields;
+ }
+
public Map<String,Object> run(Map<String,Object> args) throws Exception {
// map to inverted subset due for fetch, sort by score
Long topN = (Long)args.get(Nutch.ARG_TOPN);
@@ -187,7 +195,8 @@ public class GeneratorJob extends NutchT
numJobs = 1;
currentJobNum = 0;
currentJob = new NutchJob(getConf(), "generate: " + batchId);
- StorageUtils.initMapperJob(currentJob, FIELDS, SelectorEntry.class,
+ Collection<WebPage.Field> fields = getFields(currentJob);
+ StorageUtils.initMapperJob(currentJob, fields, SelectorEntry.class,
WebPage.class, GeneratorMapper.class, SelectorEntryPartitioner.class, true);
StorageUtils.initReducerJob(currentJob, GeneratorReducer.class);
currentJob.waitForCompletion(true);
Author: lewismc
Date: Sun May 19 21:11:54 2013
New Revision: 1484348
URL: http://svn.apache.org/r1484348
Log:
NUTCH-1573 Upgrade to most recent JUnit 4.x to improve test flexibility
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/ivy/ivy.xml
nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java
nutch/branches/2.x/src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java
nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java
nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java
nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java
nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
nutch/branches/2.x/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java
nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java
nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java
nutch/branches/2.x/src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java
nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java
nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java
nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java
nutch/branches/2.x/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java
nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java
nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java
nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java
nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java
nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java
nutch/branches/2.x/src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java
nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java
nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
nutch/branches/2.x/src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java
nutch/branches/2.x/src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java
nutch/branches/2.x/src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java
nutch/branches/2.x/src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java
nutch/branches/2.x/src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java
nutch/branches/2.x/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java
nutch/branches/2.x/src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java
nutch/branches/2.x/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java
nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java
nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestInjector.java
nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestSignatureFactory.java
nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestURLPartitioner.java
nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestUrlWithScore.java
nutch/branches/2.x/src/test/org/apache/nutch/fetcher/TestFetcher.java
nutch/branches/2.x/src/test/org/apache/nutch/indexer/TestIndexingFilters.java
nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestMetadata.java
nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java
nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLFilters.java
nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLNormalizers.java
nutch/branches/2.x/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java
nutch/branches/2.x/src/test/org/apache/nutch/parse/TestParserFactory.java
nutch/branches/2.x/src/test/org/apache/nutch/plugin/TestPluginSystem.java
nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestContent.java
nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestProtocolFactory.java
nutch/branches/2.x/src/test/org/apache/nutch/storage/TestGoraStorage.java
nutch/branches/2.x/src/test/org/apache/nutch/util/AbstractNutchTest.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestEncodingDetector.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestGZIPUtils.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestNodeWalker.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestPrefixStringMatcher.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestStringUtil.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestSuffixStringMatcher.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestTableUtil.java
nutch/branches/2.x/src/test/org/apache/nutch/util/TestURLUtil.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Sun May 19 21:11:54 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1573 Upgrade to most recent JUnit 4.x to improve test flexibility (lewismc)
+
* Added crawler-commons dependency in pom.xml (tejasp)
* NUTCH-956 solrindex issues: add field tld to Solr schema (Alexis via lewismc, snagel)
Modified: nutch/branches/2.x/ivy/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/ivy/ivy.xml?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/ivy/ivy.xml (original)
+++ nutch/branches/2.x/ivy/ivy.xml Sun May 19 21:11:54 2013
@@ -76,7 +76,7 @@
<!--Configuration: test -->
<!--artifacts needed for testing -->
- <dependency org="junit" name="junit" rev="3.8.1" conf="test->default" />
+ <dependency org="junit" name="junit" rev="4.11" conf="test->default" />
<dependency org="org.apache.hadoop" name="hadoop-test" rev="1.1.1" conf="test->default">
<exclude org="net.sf.kosmosfs" name="kfs" />
Modified: nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java (original)
+++ nutch/branches/2.x/src/plugin/creativecommons/src/test/org/creativecommons/nutch/TestCCParseFilter.java Sun May 19 21:11:54 2013
@@ -17,29 +17,24 @@
package org.creativecommons.nutch;
-import org.apache.nutch.metadata.Metadata;
-import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseUtil;
-import org.apache.nutch.protocol.Content;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.tika.mime.MimeType;
-
-import java.util.Properties;
import java.io.*;
-import java.net.URL;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestCCParseFilter extends TestCase {
+public class TestCCParseFilter {
private static final File testDir = new File(
System.getProperty("test.input"));
+ @Test
public void testPages() throws Exception {
pageTest(new File(testDir, "anchor.html"), "http://foo.com/",
"http://creativecommons.org/licenses/by-nc-sa/1.0", "a", null);
@@ -56,7 +51,6 @@ public class TestCCParseFilter extends T
public void pageTest(File file, String url, String license,
String location, String type) throws Exception {
- String contentType = "text/html";
InputStream in = new FileInputStream(file);
ByteArrayOutputStream out = new ByteArrayOutputStream(
(int) file.length());
Modified: nutch/branches/2.x/src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java (original)
+++ nutch/branches/2.x/src/plugin/feed/src/test/org/apache/nutch/parse/feed/TestFeedParser.java Sun May 19 21:11:54 2013
@@ -35,8 +35,8 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolNotFound;
import org.apache.nutch.util.NutchConfiguration;
-// Junit imports
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
*
@@ -45,7 +45,7 @@ import junit.framework.TestCase;
* Test Suite for the {@link FeedParser}.
*
*/
-public class TestFeedParser extends TestCase {
+public class TestFeedParser {
private String fileSeparator = System.getProperty("file.separator");
@@ -67,7 +67,6 @@ public class TestFeedParser extends Test
* The name of this {@link TestCase}.
*/
public TestFeedParser(String name) {
- super(name);
}
/**
@@ -85,6 +84,7 @@ public class TestFeedParser extends Test
* @throws ParseException
* If the {@link Parser}Layer cannot be loaded.
*/
+ @Test
public void testParseFetchChannel() throws ProtocolNotFound, ParseException {
String urlString;
Protocol protocol;
Modified: nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-anchor/src/test/org/apache/nutch/indexer/anchor/TestAnchorIndexingFilter.java Sun May 19 21:11:54 2013
@@ -16,14 +16,13 @@
*/
package org.apache.nutch.indexer.anchor;
-import junit.framework.TestCase;
-
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Test;
+import static org.junit.Assert.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,7 +34,7 @@ import org.slf4j.LoggerFactory;
* @author lewismc
*
*/
-public class TestAnchorIndexingFilter extends TestCase {
+public class TestAnchorIndexingFilter {
public static final Logger LOG = LoggerFactory.getLogger(TestAnchorIndexingFilter.class);
Modified: nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-basic/src/test/org/apache/nutch/indexer/basic/TestBasicIndexingFilter.java Sun May 19 21:11:54 2013
@@ -20,13 +20,12 @@ import java.nio.ByteBuffer;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
-import org.apache.nutch.fetcher.FetcherJob;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.metadata.Nutch;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Test;
-import junit.framework.TestCase;
+import static org.junit.Assert.*;
/**
* JUnit test case which tests
@@ -39,7 +38,7 @@ import junit.framework.TestCase;
* @author lewismc
*/
-public class TestBasicIndexingFilter extends TestCase {
+public class TestBasicIndexingFilter {
@Test
public void testBasicFields() throws Exception {
Modified: nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/index-more/src/test/org/apache/nutch/indexer/more/TestMoreIndexingFilter.java Sun May 19 21:11:54 2013
@@ -18,7 +18,8 @@ package org.apache.nutch.indexer.more;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -28,8 +29,9 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.EncodingDetector;
import org.apache.nutch.util.NutchConfiguration;
-public class TestMoreIndexingFilter extends TestCase {
+public class TestMoreIndexingFilter {
+ @Test
public void testContentType() throws IndexingException {
Configuration conf = NutchConfiguration.create();
assertContentType(conf, "text/html", "text/html");
@@ -45,6 +47,7 @@ public class TestMoreIndexingFilter exte
/**
* @since NUTCH-901
*/
+ @Test
public void testNoParts(){
Configuration conf = NutchConfiguration.create();
conf.setBoolean("moreIndexingFilter.indexMimeTypeParts", false);
Modified: nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java (original)
+++ nutch/branches/2.x/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/TestHTMLLanguageParser.java Sun May 19 21:11:54 2013
@@ -21,7 +21,8 @@ import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.nutch.metadata.Metadata;
@@ -32,7 +33,7 @@ import org.apache.nutch.util.EncodingDet
import org.apache.nutch.util.NutchConfiguration;
import org.apache.tika.language.LanguageIdentifier;
-public class TestHTMLLanguageParser extends TestCase {
+public class TestHTMLLanguageParser {
private static Utf8 URL = new Utf8("http://foo.bar/");
@@ -49,6 +50,7 @@ public class TestHTMLLanguageParser exte
/**
* Test parsing of language identifiers from html
**/
+ @Test
public void testMetaHTMLParsing() {
try {
@@ -71,6 +73,7 @@ public class TestHTMLLanguageParser exte
}
/** Test of <code>LanguageParser.parseLanguage(String)</code> method. */
+ @Test
public void testParseLanguage() {
String tests[][] = { { "(SCHEME=ISO.639-1) sv", "sv" },
{ "(SCHEME=RFC1766) sv-FI", "sv" }, { "(SCHEME=Z39.53) SWE", "sv" },
@@ -98,6 +101,7 @@ public class TestHTMLLanguageParser exte
}
}
+ @Test
public void testLanguageIndentifier() {
try {
long total = 0;
Modified: nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java (original)
+++ nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java Sun May 19 21:11:54 2013
@@ -17,8 +17,12 @@
package org.apache.nutch.protocol.http.api;
+import org.junit.Before;
+import org.junit.Test;
+
import crawlercommons.robots.BaseRobotRules;
-import junit.framework.TestCase;
+
+import static org.junit.Assert.*;
/**
* JUnit test case which tests
@@ -26,7 +30,7 @@ import junit.framework.TestCase;
* 2. that crawl delay is extracted correctly from the robots file
*
*/
-public class TestRobotRulesParser extends TestCase {
+public class TestRobotRulesParser {
private static final String CONTENT_TYPE = "text/plain";
private static final String SINGLE_AGENT = "Agent1";
@@ -72,14 +76,15 @@ public class TestRobotRulesParser extend
private HttpRobotRulesParser parser;
private BaseRobotRules rules;
- public TestRobotRulesParser(String name) {
- super(name);
+ @Before
+ public void setUp() {
parser = new HttpRobotRulesParser();
}
/**
* Test that the robots rules are interpreted correctly by the robots rules parser.
*/
+ @Test
public void testRobotsAgent() {
rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(), CONTENT_TYPE, SINGLE_AGENT);
@@ -104,6 +109,7 @@ public class TestRobotRulesParser extend
* Test that the crawl delay is extracted from the robots file for respective agent.
* If its not specified for a given agent, default value must be returned.
*/
+ @Test
public void testCrawlDelay() {
// for SINGLE_AGENT, the crawl delay of 10 sec ie. 10000 msec must be returned by the parser
rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(), CONTENT_TYPE, SINGLE_AGENT);
Modified: nutch/branches/2.x/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java (original)
+++ nutch/branches/2.x/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java Sun May 19 21:11:54 2013
@@ -24,23 +24,30 @@ import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
-// JUnit imports
-import junit.framework.TestCase;
+import static org.junit.Assert.*;
-// Commons Logging imports
+// Logging imports
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
// Nutch imports
import org.apache.nutch.net.URLFilter;
+//import org.apache.nutch.urlfilter.automaton.TestAutomatonURLFilter;
+//import org.apache.nutch.urlfilter.regex.TestRegexURLFilter;
+//import org.junit.runners.Suite;
+//import org.junit.runner.RunWith;
/**
* JUnit based test of class <code>RegexURLFilterBase</code>.
*
* @author Jérôme Charron
*/
-public abstract class RegexURLFilterBaseTest extends TestCase {
+
+
+//@RunWith(Suite.class)
+//@Suite.SuiteClasses({TestAutomatonURLFilter.class, TestRegexURLFilter.class})
+public abstract class RegexURLFilterBaseTest {
/** My logger */
protected static final Logger LOG = LoggerFactory.getLogger(RegexURLFilterBaseTest.class);
@@ -48,10 +55,6 @@ public abstract class RegexURLFilterBase
private final static String SEPARATOR = System.getProperty("file.separator");
private final static String SAMPLES = System.getProperty("test.data", ".");
- public RegexURLFilterBaseTest(String testName) {
- super(testName);
- }
-
protected abstract URLFilter getURLFilter(Reader rules);
protected void bench(int loops, String file) {
Modified: nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java (original)
+++ nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagIndexingFilter.java Sun May 19 21:11:54 2013
@@ -24,8 +24,7 @@ import org.apache.nutch.indexer.NutchDoc
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Test;
-
-import junit.framework.TestCase;
+import static org.junit.Assert.*;
/**
*JUnit test case for {@link RelTagIndexingFilter} which
@@ -34,7 +33,7 @@ import junit.framework.TestCase;
*@author lewismc
*/
- public class TestRelTagIndexingFilter extends TestCase {
+ public class TestRelTagIndexingFilter {
@Test
public void testRelTagFields() throws Exception {
Modified: nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java (original)
+++ nutch/branches/2.x/src/plugin/microformats-reltag/src/test/org/apache/nutch/microformats/reltag/TestRelTagParser.java Sun May 19 21:11:54 2013
@@ -32,20 +32,20 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.junit.Test;
-import junit.framework.TestCase;
+import static org.junit.Assert.*;
/**
* Junit test for {@link RelTagParser} based mainly John Xing's parser tests.
* We are not concerned with actual parse text within the sample file, instead
* we assert that the rel-tags we expect are found in the WebPage metadata.
* To check the parser is working as expected we unwrap the ByteBuffer obtained
- * from metadata, the same type as * we use in expected (String). So just the
+ * from metadata, the same type as we use in expected (String). So just the
* other way around as we wrapped the metadata value.
*
* @author lewismc
*
*/
-public class TestRelTagParser extends TestCase {
+public class TestRelTagParser {
private String fileSeparator = System.getProperty("file.separator");
@@ -61,39 +61,34 @@ public class TestRelTagParser extends Te
private Configuration conf;
- public TestRelTagParser(String name) {
- super(name);
- }
-
@Test
- public void testRelTagParser() throws ProtocolException, ParseException, IOException {
- conf = NutchConfiguration.create();
- conf.set("file.content.limit", "-1");
- Parse parse;
- String urlString = "file:" + sampleDir + fileSeparator + sampleFile;
-
- File file = new File(sampleDir + fileSeparator + sampleFile);
- byte[] bytes = new byte[(int) file.length()];
- DataInputStream in = new DataInputStream(new FileInputStream(file));
- in.readFully(bytes);
- in.close();
-
- WebPage page = new WebPage();
- page.setBaseUrl(new Utf8(urlString));
- page.setContent(ByteBuffer.wrap(bytes));
- MimeUtil mimeutil = new MimeUtil(conf);
- String mtype = mimeutil.getMimeType(file);
- page.setContentType(new Utf8(mtype));
- parse = new ParseUtil(conf).parse(urlString, page);
-
- //begin assertion for tests
- ByteBuffer bbuf = page.getFromMetadata(new Utf8("Rel-Tag"));
- byte[] byteArray = new byte[bbuf.remaining()];
- bbuf.get(byteArray);
- String s = new String(byteArray);
- //bbuf.flip();
- assertEquals("We expect 2 tab-separated rel-tag's extracted by the filter",
- expectedRelTags, s);
+ public void testRelTagParser() throws ParseException, ProtocolException, IOException {
+ conf = NutchConfiguration.create();
+ conf.set("file.content.limit", "-1");
+ Parse parse;
+ String urlString = "file:" + sampleDir + fileSeparator + sampleFile;
+
+ File file = new File(sampleDir + fileSeparator + sampleFile);
+ byte[] bytes = new byte[(int) file.length()];
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ in.readFully(bytes);
+ in.close();
+
+ WebPage page = new WebPage();
+ page.setBaseUrl(new Utf8(urlString));
+ page.setContent(ByteBuffer.wrap(bytes));
+ MimeUtil mimeutil = new MimeUtil(conf);
+ String mtype = mimeutil.getMimeType(file);
+ page.setContentType(new Utf8(mtype));
+ parse = new ParseUtil(conf).parse(urlString, page);
+ //begin assertion for tests
+ ByteBuffer bbuf = page.getFromMetadata(new Utf8("Rel-Tag"));
+ byte[] byteArray = new byte[bbuf.remaining()];
+ bbuf.get(byteArray);
+ String s = new String(byteArray);
+ //bbuf.flip();
+ assertEquals("We expect 2 tab-separated rel-tag's extracted by the filter",
+ expectedRelTags, s);
}
}
\ No newline at end of file
Modified: nutch/branches/2.x/src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-ext/src/test/org/apache/nutch/parse/ext/TestExtParser.java Sun May 19 21:11:54 2013
@@ -30,7 +30,8 @@ import org.apache.nutch.util.NutchConfig
import org.apache.hadoop.io.Text;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import java.io.File;
import java.io.FileOutputStream;
@@ -47,7 +48,7 @@ import java.io.IOException;
*
* @author John Xing
*/
-public class TestExtParser extends TestCase {
+public class TestExtParser {
private File tempFile = null;
private String urlString = null;
private Content content = null;
@@ -58,7 +59,6 @@ public class TestExtParser extends TestC
private String expectedMD5sum = "df46711a1a48caafc98b1c3b83aa1526";
public TestExtParser(String name) {
- super(name);
}
protected void setUp() throws ProtocolException, IOException {
@@ -95,6 +95,7 @@ public class TestExtParser extends TestC
// tempFile.delete();
}
+ @Test
public void testIt() throws ParseException {
String contentType;
Modified: nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java (original)
+++ nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestDOMContentUtils.java Sun May 19 21:11:54 2013
@@ -17,8 +17,6 @@
package org.apache.nutch.parse.html;
-import junit.framework.TestCase;
-
import org.apache.nutch.parse.Outlink;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
@@ -34,10 +32,14 @@ import org.xml.sax.*;
import org.w3c.dom.*;
import org.apache.html.dom.*;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Unit tests for DOMContentUtils.
*/
-public class TestDOMContentUtils extends TestCase {
+public class TestDOMContentUtils {
private static final String[] testPages= {
new String("<html><head><title> title </title><script> script </script>"
@@ -215,11 +217,8 @@ public class TestDOMContentUtils extends
private static Configuration conf;
private static DOMContentUtils utils = null;
- public TestDOMContentUtils(String name) {
- super(name);
- }
-
- private static void setup() {
+ @Before
+ public void setup() {
conf = NutchConfiguration.create();
conf.setBoolean("parser.html.form.use_action", true);
utils = new DOMContentUtils(conf);
@@ -313,6 +312,7 @@ public class TestDOMContentUtils extends
return true;
}
+ @Test
public void testGetText() {
if (testDOMs[0] == null)
setup();
@@ -328,6 +328,7 @@ public class TestDOMContentUtils extends
}
}
+ @Test
public void testGetTitle() {
if (testDOMs[0] == null)
setup();
@@ -343,6 +344,7 @@ public class TestDOMContentUtils extends
}
}
+ @Test
public void testGetOutlinks() {
if (testDOMs[0] == null)
setup();
Modified: nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java (original)
+++ nutch/branches/2.x/src/plugin/parse-html/src/test/org/apache/nutch/parse/html/TestRobotsMetaProcessor.java Sun May 19 21:11:54 2013
@@ -17,8 +17,6 @@
package org.apache.nutch.parse.html;
-import junit.framework.TestCase;
-
import org.apache.nutch.parse.HTMLMetaTags;
import java.io.ByteArrayInputStream;
@@ -29,11 +27,11 @@ import org.xml.sax.*;
import org.w3c.dom.*;
import org.apache.html.dom.*;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
/** Unit tests for HTMLMetaProcessor. */
-public class TestRobotsMetaProcessor extends TestCase {
- public TestRobotsMetaProcessor(String name) {
- super(name);
- }
+public class TestRobotsMetaProcessor {
/*
@@ -126,6 +124,7 @@ public class TestRobotsMetaProcessor ext
private URL[][] currURLsAndAnswers;
+ @Test
public void testRobotsMetaProcessor() {
DOMFragmentParser parser= new DOMFragmentParser();;
Modified: nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java (original)
+++ nutch/branches/2.x/src/plugin/parse-js/src/test/org/apache/nutch/parse/js/TestJSParseFilter.java Sun May 19 21:11:54 2013
@@ -19,7 +19,6 @@ package org.apache.nutch.parse.js;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -33,9 +32,9 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
+import org.junit.Before;
import org.junit.Test;
-
-import junit.framework.TestCase;
+import static org.junit.Assert.*;
/**
* JUnit test case for {@link JSParseFilter} which tests
@@ -45,7 +44,7 @@ import junit.framework.TestCase;
* @author lewismc
*/
-public class TestJSParseFilter extends TestCase {
+public class TestJSParseFilter {
private String fileSeparator = System.getProperty("file.separator");
@@ -58,50 +57,44 @@ public class TestJSParseFilter extends T
private Configuration conf;
- public TestJSParseFilter(String name) {
- super(name);
- }
-
- protected void setUp() {
- conf = NutchConfiguration.create();
- conf.set("file.content.limit", "-1");
+ @Before
+ public void setUp() {
+ conf = NutchConfiguration.create();
+ conf.set("file.content.limit", "-1");
}
- protected void tearDown() {
- }
-
public Outlink[] getOutlinks(String[] sampleFiles) throws ProtocolException, ParseException, IOException {
- String urlString;
- Parse parse;
+ String urlString;
+ Parse parse;
- urlString = "file:" + sampleDir + fileSeparator + sampleFiles;
- File file = new File(urlString);
- byte[] bytes = new byte[(int) file.length()];
- DataInputStream dip = new DataInputStream(new FileInputStream(file));
- dip.readFully(bytes);
- dip.close();
+ urlString = "file:" + sampleDir + fileSeparator + sampleFiles;
+ File file = new File(urlString);
+ byte[] bytes = new byte[(int) file.length()];
+ DataInputStream dip = new DataInputStream(new FileInputStream(file));
+ dip.readFully(bytes);
+ dip.close();
- WebPage page = new WebPage();
- page.setBaseUrl(new Utf8(urlString));
- page.setContent(ByteBuffer.wrap(bytes));
- MimeUtil mutil = new MimeUtil(conf);
- String mime = mutil.getMimeType(file);
- page.setContentType(new Utf8(mime));
+ WebPage page = new WebPage();
+ page.setBaseUrl(new Utf8(urlString));
+ page.setContent(ByteBuffer.wrap(bytes));
+ MimeUtil mutil = new MimeUtil(conf);
+ String mime = mutil.getMimeType(file);
+ page.setContentType(new Utf8(mime));
- parse = new ParseUtil(conf).parse(urlString, page);
- return parse.getOutlinks();
+ parse = new ParseUtil(conf).parse(urlString, page);
+ return parse.getOutlinks();
}
@Test
public void testOutlinkExtraction() throws ProtocolException, ParseException, IOException {
- String[] filenames = new File(sampleDir).list();
- for (int i = 0; i < filenames.length; i++) {
- if (filenames[i].endsWith(".js") == true) {
- assertEquals("number of outlinks in .js test file should be 5", 5, getOutlinks(sampleFiles));
- // temporarily disabled as a suitable pure JS file could not be be found.
- //} else {
- //assertEquals("number of outlinks in .html file should be X", 5, getOutlinks(sampleFiles));
- }
+ String[] filenames = new File(sampleDir).list();
+ for (int i = 0; i < filenames.length; i++) {
+ if (filenames[i].endsWith(".js") == true) {
+ assertEquals("number of outlinks in .js test file should be 5", 5, getOutlinks(sampleFiles));
+ // temporarily disabled as a suitable pure JS file could not be be found.
+ //} else {
+ //assertEquals("number of outlinks in .html file should be X", 5, getOutlinks(sampleFiles));
+ }
}
}
Modified: nutch/branches/2.x/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-swf/src/test/org/apache/nutch/parse/swf/TestSWFParser.java Sun May 19 21:11:54 2013
@@ -32,14 +32,15 @@ import org.apache.nutch.parse.ParseExcep
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* Unit tests for SWFParser.
*
* @author Andrzej Bialecki
*/
-public class TestSWFParser extends TestCase {
+public class TestSWFParser {
private String fileSeparator = System.getProperty("file.separator");
// This system property is defined in ./src/plugin/build-plugin.xml
@@ -47,10 +48,8 @@ public class TestSWFParser extends TestC
private String[] sampleFiles = new String[]{"test1.swf", "test2.swf", "test3.swf"};
private String[] sampleTexts = new String[]{"test1.txt", "test2.txt", "test3.txt"};
- private String[] texts = new String[sampleTexts.length];
-
+
public TestSWFParser(String name) {
- super(name);
for (int i = 0; i < sampleFiles.length; i++) {
try {
// read the test string
@@ -74,6 +73,7 @@ public class TestSWFParser extends TestC
protected void tearDown() {}
+ @Test
public void testIt() throws ProtocolException, ParseException {
String urlString;
Protocol protocol;
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java Sun May 19 21:11:54 2013
@@ -17,7 +17,8 @@
package org.apache.nutch.parse.tika;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.parse.tika.DOMBuilder;
@@ -43,7 +44,7 @@ import java.util.StringTokenizer;
/**
* Unit tests for DOMContentUtils.
*/
-public class DOMContentUtilsTest extends TestCase {
+public class DOMContentUtilsTest {
private static final String[] testPages = {
// 0.
@@ -200,7 +201,6 @@ public class DOMContentUtilsTest extends
public static final Logger Logger = LoggerFactory.getLogger(DOMContentUtilsTest.class);
public DOMContentUtilsTest(String name) {
- super(name);
}
private static void setup() throws Exception {
@@ -317,6 +317,7 @@ public class DOMContentUtilsTest extends
return true;
}
+ @Test
public void testGetText() throws Exception {
if (testDOMs[0] == null)
setup();
@@ -335,6 +336,7 @@ public class DOMContentUtilsTest extends
// won't work with Tika - the title is stored in the metadata but
// not put in the XHTML representation
+ @Test
public void testGetTitle() throws Exception {
if (testDOMs[0] == null)
setup();
@@ -351,6 +353,7 @@ public class DOMContentUtilsTest extends
}
}
+ @Test
public void testGetOutlinks() throws Exception {
if (testDOMs[0] == null)
setup();
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMSWordParser.java Sun May 19 21:11:54 2013
@@ -23,7 +23,9 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -34,14 +36,13 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.tika.mime.MimeType;
/**
* Unit tests for MSWordParser.
*
* @author John Xing
*/
-public class TestMSWordParser extends TestCase {
+public class TestMSWordParser {
private String fileSeparator = System.getProperty("file.separator");
// This system property is defined in ./src/plugin/build-plugin.xml
@@ -55,18 +56,12 @@ public class TestMSWordParser extends Te
private Configuration conf;
- public TestMSWordParser(String name) {
- super(name);
- }
-
- protected void setUp() {
+ @Before
+ public void setUp() {
conf = NutchConfiguration.create();
conf.set("file.content.limit", "-1");
}
- protected void tearDown() {
- }
-
public String getTextContent(String fileName) throws ProtocolException,
ParseException, IOException {
String urlString = sampleDir + fileSeparator + fileName;
@@ -89,6 +84,7 @@ public class TestMSWordParser extends Te
return parse.getText();
}
+ @Test
public void testIt() throws ProtocolException, ParseException, IOException {
for (int i = 0; i < sampleFiles.length; i++) {
String found = getTextContent(sampleFiles[i]);
@@ -97,6 +93,7 @@ public class TestMSWordParser extends Te
}
}
+ @Test
public void testOpeningDocs() throws ProtocolException, ParseException, IOException {
String[] filenames = new File(sampleDir).list();
for (int i = 0; i < filenames.length; i++) {
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestOOParser.java Sun May 19 21:11:54 2013
@@ -24,7 +24,8 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -35,92 +36,83 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.tika.mime.MimeType;
/**
* Unit tests for OOParser.
*
* @author Andrzej Bialecki
*/
-public class TestOOParser extends TestCase {
+public class TestOOParser {
- private String fileSeparator = System.getProperty("file.separator");
- // This system property is defined in ./src/plugin/build-plugin.xml
- private String sampleDir = System.getProperty("test.data", ".");
- // Make sure sample files are copied to "test.data" as specified in
- // ./src/plugin/parse-oo/build.xml during plugin compilation.
- private String[] sampleFiles = { "ootest.odt", "ootest.sxw" };
-
- private String sampleText = "ootest.txt";
-
- private String expectedText;
-
- public TestOOParser(String name) {
- super(name);
- try {
- // read the test string
- FileInputStream fis = new FileInputStream(sampleDir + fileSeparator
- + sampleText);
- StringBuffer sb = new StringBuffer();
- int len = 0;
- InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
- char[] buf = new char[1024];
- while ((len = isr.read(buf)) > 0) {
- sb.append(buf, 0, len);
- }
- isr.close();
- expectedText = sb.toString();
- // normalize space
- expectedText = expectedText.replaceAll("[ \t\r\n]+", " ");
- } catch (Exception e) {
- e.printStackTrace();
- }
+ private String fileSeparator = System.getProperty("file.separator");
+ // This system property is defined in ./src/plugin/build-plugin.xml
+ private String sampleDir = System.getProperty("test.data", ".");
+ // Make sure sample files are copied to "test.data" as specified in
+ // ./src/plugin/parse-oo/build.xml during plugin compilation.
+ private String[] sampleFiles = { "ootest.odt", "ootest.sxw" };
+
+ private String sampleText = "ootest.txt";
+
+ private String expectedText;
+
+ @Test
+ public void testIt() throws ProtocolException, ParseException, IOException {
+ String urlString;
+ Parse parse;
+ Configuration conf = NutchConfiguration.create();
+ MimeUtil mimeutil = new MimeUtil(conf);
+
+ try {
+ // read the test string
+ FileInputStream fis = new FileInputStream(sampleDir + fileSeparator
+ + sampleText);
+ StringBuffer sb = new StringBuffer();
+ int len = 0;
+ InputStreamReader isr = new InputStreamReader(fis, "UTF-8");
+ char[] buf = new char[1024];
+ while ((len = isr.read(buf)) > 0) {
+ sb.append(buf, 0, len);
+ }
+ isr.close();
+ expectedText = sb.toString();
+ // normalize space
+ expectedText = expectedText.replaceAll("[ \t\r\n]+", " ");
+ } catch (Exception e) {
+ e.printStackTrace();
}
- protected void setUp() {
- }
+ System.out.println("Expected : " + expectedText);
- protected void tearDown() {
- }
+ for (int i = 0; i < sampleFiles.length; i++) {
+ urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
+
+ if (sampleFiles[i].startsWith("ootest") == false)
+ continue;
+
+ File file = new File(sampleDir + fileSeparator + sampleFiles[i]);
+ byte[] bytes = new byte[(int) file.length()];
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ in.readFully(bytes);
+ in.close();
+
+ WebPage page = new WebPage();
+ page.setBaseUrl(new Utf8(urlString));
+ page.setContent(ByteBuffer.wrap(bytes));
+ String mtype = mimeutil.getMimeType(file);
+ page.setContentType(new Utf8(mtype));
+
+ parse = new ParseUtil(conf).parse(urlString, page);
+
+ String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim();
- public void testIt() throws ProtocolException, ParseException, IOException {
- String urlString;
- Parse parse;
- Configuration conf = NutchConfiguration.create();
- MimeUtil mimeutil = new MimeUtil(conf);
-
- System.out.println("Expected : " + expectedText);
-
- for (int i = 0; i < sampleFiles.length; i++) {
- urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];
-
- if (sampleFiles[i].startsWith("ootest") == false)
- continue;
-
- File file = new File(sampleDir + fileSeparator + sampleFiles[i]);
- byte[] bytes = new byte[(int) file.length()];
- DataInputStream in = new DataInputStream(new FileInputStream(file));
- in.readFully(bytes);
- in.close();
-
- WebPage page = new WebPage();
- page.setBaseUrl(new Utf8(urlString));
- page.setContent(ByteBuffer.wrap(bytes));
- String mtype = mimeutil.getMimeType(file);
- page.setContentType(new Utf8(mtype));
-
- parse = new ParseUtil(conf).parse(urlString, page);
-
- String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim();
-
- // simply test for the presence of a text - the ordering of the
- // elements
- // may differ from what was expected
- // in the previous tests
- assertTrue(text != null && text.length() > 0);
+ // simply test for the presence of a text - the ordering of the
+ // elements
+ // may differ from what was expected
+ // in the previous tests
+ assertTrue(text != null && text.length() > 0);
- System.out.println("Found " + sampleFiles[i] + ": " + text);
- }
+ System.out.println("Found " + sampleFiles[i] + ": " + text);
}
+ }
}
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestPdfParser.java Sun May 19 21:11:54 2013
@@ -23,7 +23,8 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -34,14 +35,13 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.tika.mime.MimeType;
/**
* Unit tests for PdfParser.
*
* @author John Xing
*/
-public class TestPdfParser extends TestCase {
+public class TestPdfParser {
private String fileSeparator = System.getProperty("file.separator");
// This system property is defined in ./src/plugin/build-plugin.xml
@@ -53,16 +53,7 @@ public class TestPdfParser extends TestC
private String expectedText = "A VERY SMALL PDF FILE";
- public TestPdfParser(String name) {
- super(name);
- }
-
- protected void setUp() {
- }
-
- protected void tearDown() {
- }
-
+ @Test
public void testIt() throws ProtocolException, ParseException, IOException {
String urlString;
Parse parse;
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRSSParser.java Sun May 19 21:11:54 2013
@@ -23,9 +23,6 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.nutch.protocol.ProtocolFactory;
-import org.apache.nutch.protocol.Protocol;
-import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.parse.Parse;
@@ -38,9 +35,8 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.tika.mime.MimeType;
-
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* Unit tests for the RSS Parser based on John Xing's TestPdfParser class.
@@ -48,7 +44,7 @@ import junit.framework.TestCase;
* @author mattmann
* @version 1.0
*/
-public class TestRSSParser extends TestCase {
+public class TestRSSParser {
private String fileSeparator = System.getProperty("file.separator");
@@ -62,18 +58,6 @@ public class TestRSSParser extends TestC
/**
* <p>
- * Default constructor
- * </p>
- *
- * @param name
- * The name of the RSSParserTest
- */
- public TestRSSParser(String name) {
- super(name);
- }
-
- /**
- * <p>
* The test method: tests out the following 2 asserts:
* </p>
*
@@ -83,9 +67,9 @@ public class TestRSSParser extends TestC
* file</li>
* </ul>
*/
+ @Test
public void testIt()throws ProtocolException, ParseException, IOException {
String urlString;
- Protocol protocol;
Parse parse;
Configuration conf = NutchConfiguration.create();
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestRTFParser.java Sun May 19 21:11:54 2013
@@ -23,7 +23,8 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -34,14 +35,13 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.tika.mime.MimeType;
/**
* Unit tests for TestRTFParser. (Adapted from John Xing msword unit tests).
*
* @author Andy Hedges
*/
-public class TestRTFParser extends TestCase {
+public class TestRTFParser {
private String fileSeparator = System.getProperty("file.separator");
// This system property is defined in ./src/plugin/build-plugin.xml
@@ -51,16 +51,7 @@ public class TestRTFParser extends TestC
// Check ./src/plugin/parse-rtf/sample/README.txt for what they are.
private String rtfFile = "test.rtf";
- public TestRTFParser(String name) {
- super(name);
- }
-
- protected void setUp() {
- }
-
- protected void tearDown() {
- }
-
+ @Test
public void testIt() throws ProtocolException, ParseException, IOException {
String urlString;
Modified: nutch/branches/2.x/src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java (original)
+++ nutch/branches/2.x/src/plugin/parse-zip/src/test/org/apache/nutch/parse/zip/TestZipParser.java Sun May 19 21:11:54 2013
@@ -29,15 +29,16 @@ import org.apache.hadoop.conf.Configurat
import org.apache.nutch.util.NutchConfiguration;
import org.apache.hadoop.io.Text;
+import org.junit.Test;
-import junit.framework.TestCase;
+import static org.junit.Assert.*;
/**
* Based on Unit tests for MSWordParser by John Xing
*
* @author Rohit Kulkarni & Ashish Vaidya
*/
-public class TestZipParser extends TestCase {
+public class TestZipParser {
private String fileSeparator = System.getProperty("file.separator");
// This system property is defined in ./src/plugin/build-plugin.xml
@@ -50,13 +51,13 @@ public class TestZipParser extends TestC
private String expectedText = "textfile.txt This is text file number 1 ";
public TestZipParser(String name) {
- super(name);
}
protected void setUp() {}
protected void tearDown() {}
+ @Test
public void testIt() throws ProtocolException, ParseException {
String urlString;
Protocol protocol;
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/test/org/apache/nutch/protocol/file/TestProtocolFile.java Sun May 19 21:11:54 2013
@@ -18,10 +18,8 @@
package org.apache.nutch.protocol.file;
// Hadoop imports
-import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.Text;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolException;
@@ -31,6 +29,9 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolStatusCodes;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* @author mattmann
@@ -40,7 +41,7 @@ import org.apache.nutch.util.NutchConfig
* Unit tests for the {@link File}Protocol.
* </p>.
*/
-public class TestProtocolFile extends TestCase {
+public class TestProtocolFile {
private String fileSeparator = System.getProperty("file.separator");
private String sampleDir = System.getProperty("test.data", ".");
@@ -52,10 +53,12 @@ public class TestProtocolFile extends Te
private Configuration conf;
- protected void setUp() {
+ @Before
+ public void setUp() {
conf = NutchConfiguration.create();
}
+ @Test
public void testSetContentType() throws ProtocolException {
for (String testTextFile : testTextFiles) {
setContentType(testTextFile);
Modified: nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java (original)
+++ nutch/branches/2.x/src/plugin/protocol-httpclient/src/test/org/apache/nutch/protocol/httpclient/TestProtocolHttpClient.java Sun May 19 21:11:54 2013
@@ -20,7 +20,10 @@ package org.apache.nutch.protocol.httpcl
import java.net.MalformedURLException;
import java.net.URL;
-import junit.framework.TestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
@@ -35,7 +38,7 @@ import org.mortbay.jetty.servlet.Servlet
*
* @author Susam Pal
*/
-public class TestProtocolHttpClient extends TestCase {
+public class TestProtocolHttpClient {
private Server server;
private Configuration conf;
@@ -43,7 +46,8 @@ public class TestProtocolHttpClient exte
private int port;
private Http http = new Http();
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
server = new Server();
@@ -69,7 +73,8 @@ public class TestProtocolHttpClient exte
http.setConf(conf);
}
- protected void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
server.stop();
}
@@ -79,6 +84,7 @@ public class TestProtocolHttpClient exte
* @throws Exception
* If an error occurs or the test case fails.
*/
+ @Test
public void testCookies() throws Exception {
startServer(47500);
fetchPage("/cookies.jsp", 200);
@@ -92,6 +98,7 @@ public class TestProtocolHttpClient exte
* @throws Exception
* If an error occurs or the test case fails.
*/
+ @Test
public void testNoPreemptiveAuth() throws Exception {
startServer(47500);
fetchPage("/noauth.jsp", 200);
@@ -104,6 +111,7 @@ public class TestProtocolHttpClient exte
* @throws Exception
* If an error occurs or the test case fails.
*/
+ @Test
public void testDefaultCredentials() throws Exception {
startServer(47502);
fetchPage("/basic.jsp", 200);
@@ -116,6 +124,7 @@ public class TestProtocolHttpClient exte
* @throws Exception
* If an error occurs or the test case fails.
*/
+ @Test
public void testBasicAuth() throws Exception {
startServer(47500);
fetchPage("/basic.jsp", 200);
@@ -132,6 +141,7 @@ public class TestProtocolHttpClient exte
* @throws Exception
* If an error occurs or the test case fails.
*/
+ @Test
public void testOtherRealmsNoAuth() throws Exception {
startServer(47501);
fetchPage("/basic.jsp", 200);
@@ -146,6 +156,7 @@ public class TestProtocolHttpClient exte
* @throws Exception
* If an error occurs or the test case fails.
*/
+ @Test
public void testDigestAuth() throws Exception {
startServer(47500);
fetchPage("/digest.jsp", 200);
@@ -158,6 +169,7 @@ public class TestProtocolHttpClient exte
* @throws Exception
* If an error occurs or the test case fails.
*/
+ @Test
public void testNtlmAuth() throws Exception {
startServer(47501);
fetchPage("/ntlm.jsp", 200);
Modified: nutch/branches/2.x/src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java (original)
+++ nutch/branches/2.x/src/plugin/subcollection/src/test/org/apache/nutch/collection/TestSubcollection.java Sun May 19 21:11:54 2013
@@ -22,14 +22,16 @@ import java.util.Collection;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestSubcollection extends TestCase {
+public class TestSubcollection {
/**Test filtering logic
*
* @throws Exception
*/
+ @Test
public void testFilter() throws Exception {
Subcollection sc=new Subcollection(NutchConfiguration.create());
sc.setWhiteList("www.nutch.org\nwww.apache.org");
@@ -46,6 +48,7 @@ public class TestSubcollection extends T
assertEquals(null, sc.filter("http://www.google.com/"));
}
+ @Test
public void testInput(){
StringBuffer xml=new StringBuffer();
xml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
Modified: nutch/branches/2.x/src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java (original)
+++ nutch/branches/2.x/src/plugin/urlfilter-automaton/src/test/org/apache/nutch/urlfilter/automaton/TestAutomatonURLFilter.java Sun May 19 21:11:54 2013
@@ -20,14 +20,12 @@ package org.apache.nutch.urlfilter.autom
import java.io.IOException;
import java.io.Reader;
-// JUnit imports
-import junit.framework.Test;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.nutch.net.*;
// Nutch imports
import org.apache.nutch.urlfilter.api.RegexURLFilterBaseTest;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
@@ -37,18 +35,6 @@ import org.apache.nutch.urlfilter.api.Re
*/
public class TestAutomatonURLFilter extends RegexURLFilterBaseTest {
- public TestAutomatonURLFilter(String testName) {
- super(testName);
- }
-
- public static Test suite() {
- return new TestSuite(TestAutomatonURLFilter.class);
- }
-
- public static void main(String[] args) {
- TestRunner.run(suite());
- }
-
protected URLFilter getURLFilter(Reader rules) {
try {
return new AutomatonURLFilter(rules);
@@ -58,6 +44,7 @@ public class TestAutomatonURLFilter exte
}
}
+ @Test
public void test() {
test("WholeWebCrawling");
test("IntranetCrawling");
Modified: nutch/branches/2.x/src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java (original)
+++ nutch/branches/2.x/src/plugin/urlfilter-domain/src/test/org/apache/nutch/urlfilter/domain/TestDomainURLFilter.java Sun May 19 21:11:54 2013
@@ -16,25 +16,22 @@
*/
package org.apache.nutch.urlfilter.domain;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-public class TestDomainURLFilter
- extends TestCase {
+public class TestDomainURLFilter {
protected static final Logger LOG = LoggerFactory.getLogger(TestDomainURLFilter.class);
private final static String SEPARATOR = System.getProperty("file.separator");
private final static String SAMPLES = System.getProperty("test.data", ".");
- public TestDomainURLFilter(String testName) {
- super(testName);
- }
-
+ @Test
public void testFilter()
throws Exception {
Modified: nutch/branches/2.x/src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java (original)
+++ nutch/branches/2.x/src/plugin/urlfilter-regex/src/test/org/apache/nutch/urlfilter/regex/TestRegexURLFilter.java Sun May 19 21:11:54 2013
@@ -20,15 +20,11 @@ package org.apache.nutch.urlfilter.regex
import java.io.IOException;
import java.io.Reader;
-// JUnit imports
-import junit.framework.Test;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.nutch.net.*;
-
-// Nutch imports
import org.apache.nutch.urlfilter.api.RegexURLFilterBaseTest;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* JUnit based test of class <code>RegexURLFilter</code>.
@@ -37,18 +33,6 @@ import org.apache.nutch.urlfilter.api.Re
*/
public class TestRegexURLFilter extends RegexURLFilterBaseTest {
- public TestRegexURLFilter(String testName) {
- super(testName);
- }
-
- public static Test suite() {
- return new TestSuite(TestRegexURLFilter.class);
- }
-
- public static void main(String[] args) {
- TestRunner.run(suite());
- }
-
protected URLFilter getURLFilter(Reader rules) {
try {
return new RegexURLFilter(rules);
@@ -58,6 +42,7 @@ public class TestRegexURLFilter extends
}
}
+ @Test
public void test() {
test("WholeWebCrawling");
test("IntranetCrawling");
Modified: nutch/branches/2.x/src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java (original)
+++ nutch/branches/2.x/src/plugin/urlfilter-suffix/src/test/org/apache/nutch/urlfilter/suffix/TestSuffixURLFilter.java Sun May 19 21:11:54 2013
@@ -19,10 +19,9 @@ package org.apache.nutch.urlfilter.suffi
import java.io.IOException;
import java.io.StringReader;
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
@@ -30,7 +29,7 @@ import junit.textui.TestRunner;
*
* @author Andrzej Bialecki
*/
-public class TestSuffixURLFilter extends TestCase {
+public class TestSuffixURLFilter {
private static final String suffixes =
"# this is a comment\n" +
"\n" +
@@ -117,22 +116,12 @@ public class TestSuffixURLFilter extends
private SuffixURLFilter filter = null;
- public TestSuffixURLFilter(String testName) {
- super(testName);
- }
-
- public static Test suite() {
- return new TestSuite(TestSuffixURLFilter.class);
- }
-
- public static void main(String[] args) {
- TestRunner.run(suite());
- }
-
+ @Before
public void setUp() throws IOException {
filter = new SuffixURLFilter(new StringReader(suffixes));
}
+ @Test
public void testModeAccept() {
filter.setIgnoreCase(false);
filter.setModeAccept(true);
@@ -141,6 +130,7 @@ public class TestSuffixURLFilter extends
}
}
+ @Test
public void testModeReject() {
filter.setIgnoreCase(false);
filter.setModeAccept(false);
@@ -149,6 +139,7 @@ public class TestSuffixURLFilter extends
}
}
+ @Test
public void testModeAcceptIgnoreCase() {
filter.setIgnoreCase(true);
filter.setModeAccept(true);
@@ -157,6 +148,7 @@ public class TestSuffixURLFilter extends
}
}
+ @Test
public void testModeRejectIgnoreCase() {
filter.setIgnoreCase(true);
filter.setModeAccept(false);
@@ -165,6 +157,7 @@ public class TestSuffixURLFilter extends
}
}
+ @Test
public void testModeAcceptAndNonPathFilter() {
filter.setModeAccept(true);
filter.setFilterFromPath(false);
@@ -173,6 +166,7 @@ public class TestSuffixURLFilter extends
}
}
+ @Test
public void testModeAcceptAndPathFilter() {
filter.setModeAccept(true);
filter.setFilterFromPath(true);
Modified: nutch/branches/2.x/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java (original)
+++ nutch/branches/2.x/src/plugin/urlnormalizer-basic/src/test/org/apache/nutch/net/urlnormalizer/basic/TestBasicURLNormalizer.java Sun May 19 21:11:54 2013
@@ -21,20 +21,23 @@ import org.apache.hadoop.conf.Configurat
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
/** Unit tests for BasicURLNormalizer. */
-public class TestBasicURLNormalizer extends TestCase {
+public class TestBasicURLNormalizer {
private BasicURLNormalizer normalizer;
private Configuration conf;
- public TestBasicURLNormalizer(String name) {
- super(name);
+ @Before
+ public void setUp() {
normalizer = new BasicURLNormalizer();
conf = NutchConfiguration.create();
normalizer.setConf(conf);
}
+ @Test
public void testNormalizer() throws Exception {
// check that leading and trailing spaces are removed
normalizeTest(" http://foo.com/ ", "http://foo.com/");
@@ -109,11 +112,4 @@ public class TestBasicURLNormalizer exte
assertEquals(normal, normalizer.normalize(weird, URLNormalizers.SCOPE_DEFAULT));
}
- public static void main(String[] args) throws Exception {
- new TestBasicURLNormalizer("test").testNormalizer();
- }
-
-
-
-
}
\ No newline at end of file
Modified: nutch/branches/2.x/src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java (original)
+++ nutch/branches/2.x/src/plugin/urlnormalizer-pass/src/test/org/apache/nutch/net/urlnormalizer/pass/TestPassURLNormalizer.java Sun May 19 21:11:54 2013
@@ -23,10 +23,12 @@ import org.apache.hadoop.conf.Configurat
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestPassURLNormalizer extends TestCase {
+public class TestPassURLNormalizer {
+ @Test
public void testPassURLNormalizer() {
Configuration conf = NutchConfiguration.create();
Modified: nutch/branches/2.x/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java (original)
+++ nutch/branches/2.x/src/plugin/urlnormalizer-regex/src/test/org/apache/nutch/net/urlnormalizer/regex/TestRegexURLNormalizer.java Sun May 19 21:11:54 2013
@@ -24,22 +24,22 @@ import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
-
/** Unit tests for RegexUrlNormalizer. */
-public class TestRegexURLNormalizer extends TestCase {
+public class TestRegexURLNormalizer {
private static final Logger LOG = LoggerFactory.getLogger(TestRegexURLNormalizer.class);
private RegexURLNormalizer normalizer;
@@ -51,8 +51,8 @@ public class TestRegexURLNormalizer exte
// Make sure sample files are copied to "test.data" as specified in
// ./src/plugin/urlnormalizer-regex/build.xml during plugin compilation.
- public TestRegexURLNormalizer(String name) throws IOException {
- super(name);
+ @Before
+ public void setUp() throws IOException {
normalizer = new RegexURLNormalizer();
conf = NutchConfiguration.create();
normalizer.setConf(conf);
@@ -77,11 +77,13 @@ public class TestRegexURLNormalizer exte
}
}
+ @Test
public void testNormalizerDefault() throws Exception {
normalizeTest((NormalizedURL[])testData.get(URLNormalizers.SCOPE_DEFAULT),
URLNormalizers.SCOPE_DEFAULT);
}
+ @Test
public void testNormalizerScope() throws Exception {
Iterator it = testData.keySet().iterator();
while (it.hasNext()) {
@@ -139,43 +141,4 @@ public class TestRegexURLNormalizer exte
}
return (NormalizedURL[]) list.toArray(new NormalizedURL[list.size()]);
}
-
- public static void main(String[] args) throws Exception {
- if (args.length == 0) {
- System.err.println("TestRegexURLNormalizer [-bench <iter>] <scope>");
- System.exit(-1);
- }
- boolean bench = false;
- int iter = -1;
- String scope = null;
- for (int i = 0; i < args.length; i++) {
- if (args[i].equals("-bench")) {
- bench = true;
- iter = Integer.parseInt(args[++i]);
- } else scope = args[i];
- }
- if (scope == null) {
- System.err.println("Missing required scope name.");
- System.exit(-1);
- }
- if (bench && iter < 0) {
- System.err.println("Invalid number of iterations: " + iter);
- System.exit(-1);
- }
- TestRegexURLNormalizer test = new TestRegexURLNormalizer("test");
- NormalizedURL[] urls = (NormalizedURL[])test.testData.get(scope);
- if (urls == null) {
- LOG.warn("Missing test data for scope '" + scope + "', using default scope.");
- scope = URLNormalizers.SCOPE_DEFAULT;
- urls = (NormalizedURL[])test.testData.get(scope);
- }
- if (bench) {
- test.bench(iter, scope);
- } else {
- test.normalizeTest(urls, scope);
- }
- }
-
-
-
}
Modified: nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java Sun May 19 21:11:54 2013
@@ -28,15 +28,16 @@ import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.AbstractNutchTest;
import org.apache.nutch.util.CrawlTestUtil;
import org.apache.nutch.util.TableUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* Basic generator test. 1. Insert entries in webtable 2. Generates entries to
* fetch 3. Verifies that number of generated urls match 4. Verifies that
* highest scoring urls are generated
*
- * @author nutch-dev <nutch-dev at lucene.apache.org>
- * @param <URLWebPage>
- *
*/
public class TestGenerator extends AbstractNutchTest {
@@ -46,12 +47,25 @@ public class TestGenerator extends Abstr
WebPage.Field.MARKERS.getName(),
WebPage.Field.SCORE.getName()
};
+
+ @Override
+ @Before
+ public void setUp() throws Exception{
+ super.setUp();
+ }
+
+ @Override
+ @After
+ public void tearDown()throws Exception {
+ super.tearDown();
+ }
/**
* Test that generator generates fetchlist ordered by score (desc).
*
* @throws Exception
*/
+ @Test
public void testGenerateHighest() throws Exception {
final int NUM_RESULTS = 2;
@@ -111,6 +125,7 @@ public class TestGenerator extends Abstr
*
* @throws Exception
*/
+ @Test
public void testGenerateHostLimit() throws Exception {
ArrayList<URLWebPage> list = new ArrayList<URLWebPage>();
@@ -158,6 +173,7 @@ public class TestGenerator extends Abstr
*
* @throws Exception
*/
+ @Test
public void testGenerateDomainLimit() throws Exception {
ArrayList<URLWebPage> list = new ArrayList<URLWebPage>();
@@ -209,6 +225,7 @@ public class TestGenerator extends Abstr
* @throws Exception
* @throws IOException
*/
+ @Test
public void testFilter() throws IOException, Exception {
ArrayList<URLWebPage> list = new ArrayList<URLWebPage>();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestInjector.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestInjector.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestInjector.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestInjector.java Sun May 19 21:11:54 2013
@@ -28,6 +28,8 @@ import org.apache.nutch.util.AbstractNut
import org.apache.nutch.util.CrawlTestUtil;
import org.apache.gora.util.ByteUtils;
import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* Basic injector test: 1. Creates a text file with urls 2. Injects them into
@@ -46,6 +48,7 @@ public class TestInjector extends Abstra
urlPath = new Path(testdir, "urls");
}
+ @Test
public void testInject() throws Exception {
ArrayList<String> urls = new ArrayList<String>();
for (int i = 0; i < 100; i++) {
Modified: nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestSignatureFactory.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestSignatureFactory.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestSignatureFactory.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestSignatureFactory.java Sun May 19 21:11:54 2013
@@ -19,10 +19,12 @@ package org.apache.nutch.crawl;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestSignatureFactory extends TestCase {
+public class TestSignatureFactory {
+ @Test
public void testGetSignature() {
Configuration conf=NutchConfiguration.create();
Signature signature1=SignatureFactory.getSignature(conf);
Modified: nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestURLPartitioner.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestURLPartitioner.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestURLPartitioner.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestURLPartitioner.java Sun May 19 21:11:54 2013
@@ -18,7 +18,8 @@ package org.apache.nutch.crawl;
import java.net.MalformedURLException;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
@@ -33,11 +34,12 @@ import org.apache.nutch.util.TableUtil;
/**
* Tests {@link URLPartitioner}
*/
-public class TestURLPartitioner extends TestCase {
+public class TestURLPartitioner {
/**
* tests one reducer, everything goes into one partition, using host partitioner.
*/
+ @Test
public void testOneReducer() {
URLPartitioner partitioner = new URLPartitioner();
Configuration conf = NutchConfiguration.create();
@@ -53,6 +55,7 @@ public class TestURLPartitioner extends
/**
* tests partitioning by host
*/
+ @Test
public void testModeHost() {
URLPartitioner partitioner = new URLPartitioner();
Configuration conf = NutchConfiguration.create();
@@ -75,6 +78,7 @@ public class TestURLPartitioner extends
/**
* tests partitioning by domain
*/
+ @Test
public void testModeDomain() {
URLPartitioner partitioner = new URLPartitioner();
Configuration conf = NutchConfiguration.create();
@@ -97,6 +101,7 @@ public class TestURLPartitioner extends
/**
* tests partitioning by IP
*/
+ @Test
public void testModeIP() {
URLPartitioner partitioner = new URLPartitioner();
Configuration conf = NutchConfiguration.create();
@@ -117,10 +122,10 @@ public class TestURLPartitioner extends
partitionWithoutWWW, partitionWithWWW);
}
-
/**
* Test the seed functionality, using host partitioner.
*/
+ @Test
public void testSeed() {
URLPartitioner partitioner = new URLPartitioner();
Configuration conf = NutchConfiguration.create();
@@ -143,6 +148,7 @@ public class TestURLPartitioner extends
/**
* Tests the {@link SelectorEntryPartitioner}.
*/
+ @Test
public void testSelectorEntryPartitioner() {
//The reference partitioner
URLPartitioner refPartitioner = new URLPartitioner();
@@ -174,6 +180,7 @@ public class TestURLPartitioner extends
* Tests the {@link FetchEntryPartitioner}
* @throws MalformedURLException
*/
+ @Test
public void testFetchEntryPartitioner() throws MalformedURLException {
//The reference partitioner
URLPartitioner refPartitioner = new URLPartitioner();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestUrlWithScore.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestUrlWithScore.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestUrlWithScore.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestUrlWithScore.java Sun May 19 21:11:54 2013
@@ -22,7 +22,8 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.hadoop.io.RawComparator;
import org.apache.nutch.crawl.UrlWithScore;
@@ -33,8 +34,9 @@ import org.apache.nutch.crawl.UrlWithSco
/**
* Tests {@link UrlWithScore} with serialization, partitioning and sorting.
*/
-public class TestUrlWithScore extends TestCase {
+public class TestUrlWithScore {
+ @Test
public void testSerialization() throws IOException {
// create a key and test basic functionality
UrlWithScore keyOut = new UrlWithScore("http://example.org/", 1f);
@@ -58,6 +60,7 @@ public class TestUrlWithScore extends Te
out.close();
}
+ @Test
public void testPartitioner() throws IOException {
UrlOnlyPartitioner part = new UrlOnlyPartitioner();
@@ -80,6 +83,7 @@ public class TestUrlWithScore extends Te
assertEquals(partForKey3, part.getPartition(k5, null, numReduces));
}
+ @Test
public void testUrlOnlySorting() throws IOException {
UrlOnlyComparator comp = new UrlOnlyComparator();
@@ -104,6 +108,7 @@ public class TestUrlWithScore extends Te
assertEquals(1, compareBothRegularAndRaw(comp, k5, k1));
}
+ @Test
public void testUrlScoreSorting() throws IOException {
UrlScoreComparator comp = new UrlScoreComparator();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/fetcher/TestFetcher.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/fetcher/TestFetcher.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/fetcher/TestFetcher.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/fetcher/TestFetcher.java Sun May 19 21:11:54 2013
@@ -30,6 +30,11 @@ import org.apache.nutch.util.AbstractNut
import org.apache.nutch.util.CrawlTestUtil;
import org.mortbay.jetty.Server;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Basic fetcher test
* 1. generate seedlist
@@ -37,7 +42,6 @@ import org.mortbay.jetty.Server;
* 3. generate
* 3. fetch
* 4. Verify contents
- * @author nutch-dev <nutch-dev at lucene.apache.org>
*
*/
public class TestFetcher extends AbstractNutchTest {
@@ -47,6 +51,7 @@ public class TestFetcher extends Abstrac
Server server;
@Override
+ @Before
public void setUp() throws Exception{
super.setUp();
urlPath = new Path(testdir, "urls");
@@ -55,11 +60,13 @@ public class TestFetcher extends Abstrac
}
@Override
+ @After
public void tearDown() throws Exception{
server.stop();
fs.delete(testdir, true);
}
+ @Test
public void testFetch() throws Exception {
//generate seedlist
@@ -124,6 +131,7 @@ public class TestFetcher extends Abstrac
urls.add("http://127.0.0.1:" + server.getConnectors()[0].getPort() + "/" + page);
}
+ @Test
public void testAgentNameCheck() {
boolean failedNoAgentName = false;
Modified: nutch/branches/2.x/src/test/org/apache/nutch/indexer/TestIndexingFilters.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/indexer/TestIndexingFilters.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/indexer/TestIndexingFilters.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/indexer/TestIndexingFilters.java Sun May
19 21:11:54 2013
@@ -16,7 +16,8 @@
*/
package org.apache.nutch.indexer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
@@ -24,12 +25,13 @@ import org.apache.nutch.metadata.Metadat
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
-public class TestIndexingFilters extends TestCase {
+public class TestIndexingFilters {
/**
* Test behaviour when defined filter does not exist.
* @throws IndexingException
*/
+ @Test
public void testNonExistingIndexingFilter() throws IndexingException {
Configuration conf = NutchConfiguration.create();
conf.addResource("nutch-default.xml");
@@ -50,6 +52,7 @@ public class TestIndexingFilters extends
* Test behaviour when NutchDOcument is null
* @throws IndexingException
*/
+ @Test
public void testNutchDocumentNullIndexingFilter() throws IndexingException{
Configuration conf = NutchConfiguration.create();
conf.addResource("nutch-default.xml");
@@ -64,38 +67,38 @@ public class TestIndexingFilters extends
assertNull(doc);
}
- /**
- * Test behaviour when reset the index filter order will not take effect
- *
- * @throws IndexingException
- */
- public void testFilterCacheIndexingFilter() throws IndexingException{
- Configuration conf = NutchConfiguration.create();
- conf.addResource("nutch-default.xml");
- conf.addResource("crawl-tests.xml");
-
- String class1 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
- conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1);
-
- IndexingFilters filters1 = new IndexingFilters(conf);
- WebPage page = new WebPage();
- page.setText(new Utf8("text"));
- page.setTitle(new Utf8("title"));
- NutchDocument fdoc1 = filters1.filter(new NutchDocument(),"http://www.example.com/",page);
-
- // add another index filter
- String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer";
- // set content metadata
- Metadata md = new Metadata();
- md.add("example","data");
- // set content metadata property defined in MetadataIndexer
- conf.set("index.content.md","example");
- // add MetadataIndxer filter
- conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);
- IndexingFilters filters2 = new IndexingFilters(conf);
- NutchDocument fdoc2 = filters2.filter(new NutchDocument(),"http://www.example.com/",page);
- assertEquals(fdoc1.getFieldNames().size(),fdoc2.getFieldNames().size());
- }
+ /**
+ * Test behaviour when reset the index filter order will not take effect
+ *
+ * @throws IndexingException
+ */
+ @Test
+ public void testFilterCacheIndexingFilter() throws IndexingException{
+ Configuration conf = NutchConfiguration.create();
+ conf.addResource("nutch-default.xml");
+ conf.addResource("crawl-tests.xml");
+
+ String class1 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
+ conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1);
+ IndexingFilters filters1 = new IndexingFilters(conf);
+ WebPage page = new WebPage();
+ page.setText(new Utf8("text"));
+ page.setTitle(new Utf8("title"));
+ NutchDocument fdoc1 = filters1.filter(new NutchDocument(),"http://www.example.com/",page);
+
+ // add another index filter
+ String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer";
+ // set content metadata
+ Metadata md = new Metadata();
+ md.add("example","data");
+ // set content metadata property defined in MetadataIndexer
+ conf.set("index.content.md","example");
+ // add MetadataIndxer filter
+ conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);
+ IndexingFilters filters2 = new IndexingFilters(conf);
+ NutchDocument fdoc2 = filters2.filter(new NutchDocument(),"http://www.example.com/",page);
+ assertEquals(fdoc1.getFieldNames().size(),fdoc2.getFieldNames().size());
+ }
}
Modified: nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestMetadata.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestMetadata.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestMetadata.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestMetadata.java Sun May 19 21:11:54
2013
@@ -22,30 +22,16 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Properties;
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* JUnit based tests of class {@link org.apache.nutch.metadata.Metadata}.
*/
-public class TestMetadata extends TestCase {
+public class TestMetadata {
private static final String CONTENTTYPE = "contenttype";
- public TestMetadata(String testName) {
- super(testName);
- }
-
- public static Test suite() {
- return new TestSuite(TestMetadata.class);
- }
-
- public static void main(String[] args) {
- TestRunner.run(suite());
- }
-
/**
* Test to ensure that only non-null values get written when the
* {@link Metadata} object is written using a Writeable.
@@ -53,6 +39,7 @@ public class TestMetadata extends TestCa
* @since NUTCH-406
*
*/
+ @Test
public void testWriteNonNull() {
Metadata met = new Metadata();
met.add(CONTENTTYPE, null);
@@ -83,6 +70,7 @@ public class TestMetadata extends TestCa
}
/** Test for the <code>add(String, String)</code> method. */
+ @Test
public void testAdd() {
String[] values = null;
Metadata meta = new Metadata();
@@ -112,6 +100,7 @@ public class TestMetadata extends TestCa
}
/** Test for the <code>set(String, String)</code> method. */
+ @Test
public void testSet() {
String[] values = null;
Metadata meta = new Metadata();
@@ -138,6 +127,7 @@ public class TestMetadata extends TestCa
}
/** Test for <code>setAll(Properties)</code> method. */
+ @Test
public void testSetProperties() {
String[] values = null;
Metadata meta = new Metadata();
@@ -165,6 +155,7 @@ public class TestMetadata extends TestCa
}
/** Test for <code>get(String)</code> method. */
+ @Test
public void testGet() {
Metadata meta = new Metadata();
assertNull(meta.get("a-name"));
@@ -175,6 +166,7 @@ public class TestMetadata extends TestCa
}
/** Test for <code>isMultiValued()</code> method. */
+ @Test
public void testIsMultiValued() {
Metadata meta = new Metadata();
assertFalse(meta.isMultiValued("key"));
@@ -185,6 +177,7 @@ public class TestMetadata extends TestCa
}
/** Test for <code>names</code> method. */
+ @Test
public void testNames() {
String[] names = null;
Metadata meta = new Metadata();
@@ -201,6 +194,7 @@ public class TestMetadata extends TestCa
}
/** Test for <code>remove(String)</code> method. */
+ @Test
public void testRemove() {
Metadata meta = new Metadata();
meta.remove("name-one");
@@ -222,6 +216,7 @@ public class TestMetadata extends TestCa
}
/** Test for <code>equals(Object)</code> method. */
+ @Test
public void testObject() {
Metadata meta1 = new Metadata();
Metadata meta2 = new Metadata();
@@ -247,6 +242,7 @@ public class TestMetadata extends TestCa
}
/** Test for <code>Writable</code> implementation. */
+ @Test
public void testWritable() {
Metadata result = null;
Metadata meta = new Metadata();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/metadata/TestSpellCheckedMetadata.java Sun
May 19 21:11:54 2013
@@ -22,10 +22,8 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Properties;
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* JUnit based tests of class
@@ -34,23 +32,13 @@ import junit.textui.TestRunner;
* @author Chris Mattmann
* @author Jérôme Charron
*/
-public class TestSpellCheckedMetadata extends TestCase {
- private static final int NUM_ITERATIONS = 10000;
-
- public TestSpellCheckedMetadata(String testName) {
- super(testName);
- }
+public class TestSpellCheckedMetadata {
- public static Test suite() {
- return new TestSuite(TestSpellCheckedMetadata.class);
- }
-
- public static void main(String[] args) {
- TestRunner.run(suite());
- }
+ private static final int NUM_ITERATIONS = 10000;
/** Test for the <code>getNormalizedName(String)</code> method. */
+ @Test
public void testGetNormalizedName() {
assertEquals("Content-Type", SpellCheckedMetadata
.getNormalizedName("Content-Type"));
@@ -67,6 +55,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for the <code>add(String, String)</code> method. */
+ @Test
public void testAdd() {
String[] values = null;
SpellCheckedMetadata meta = new SpellCheckedMetadata();
@@ -96,6 +85,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for the <code>set(String, String)</code> method. */
+ @Test
public void testSet() {
String[] values = null;
SpellCheckedMetadata meta = new SpellCheckedMetadata();
@@ -122,6 +112,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for <code>setAll(Properties)</code> method. */
+ @Test
public void testSetProperties() {
String[] values = null;
SpellCheckedMetadata meta = new SpellCheckedMetadata();
@@ -149,6 +140,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for <code>get(String)</code> method. */
+ @Test
public void testGet() {
SpellCheckedMetadata meta = new SpellCheckedMetadata();
assertNull(meta.get("a-name"));
@@ -160,6 +152,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for <code>isMultiValued()</code> method. */
+ @Test
public void testIsMultiValued() {
SpellCheckedMetadata meta = new SpellCheckedMetadata();
assertFalse(meta.isMultiValued("key"));
@@ -170,6 +163,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for <code>names</code> method. */
+ @Test
public void testNames() {
String[] names = null;
SpellCheckedMetadata meta = new SpellCheckedMetadata();
@@ -186,6 +180,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for <code>remove(String)</code> method. */
+ @Test
public void testRemove() {
SpellCheckedMetadata meta = new SpellCheckedMetadata();
meta.remove("name-one");
@@ -207,6 +202,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for <code>equals(Object)</code> method. */
+ @Test
public void testObject() {
SpellCheckedMetadata meta1 = new SpellCheckedMetadata();
SpellCheckedMetadata meta2 = new SpellCheckedMetadata();
@@ -232,6 +228,7 @@ public class TestSpellCheckedMetadata ex
}
/** Test for <code>Writable</code> implementation. */
+ @Test
public void testWritable() {
SpellCheckedMetadata result = null;
SpellCheckedMetadata meta = new SpellCheckedMetadata();
@@ -259,6 +256,7 @@ public class TestSpellCheckedMetadata ex
* IO Test method, usable only when you plan to do changes in metadata
* to measure relative performance impact.
*/
+ @Test
public final void testHandlingSpeed() {
SpellCheckedMetadata result;
long start = System.currentTimeMillis();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLFilters.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLFilters.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLFilters.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLFilters.java Sun May 19 21:11:54
2013
@@ -19,14 +19,16 @@ package org.apache.nutch.net;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestURLFilters extends TestCase {
+public class TestURLFilters {
/**
* Testcase for NUTCH-325.
* @throws URLFilterException
*/
+ @Test
public void testNonExistingUrlFilter() throws URLFilterException {
Configuration conf = NutchConfiguration.create();
String class1 = "NonExistingFilter";
@@ -34,7 +36,12 @@ public class TestURLFilters extends Test
conf.set(URLFilters.URLFILTER_ORDER, class1 + " " + class2);
URLFilters normalizers = new URLFilters(conf);
- normalizers.filter("http://someurl/");
+ assertNotNull(normalizers);
+ try {
+ normalizers.filter("http://someurl/");
+ } catch (URLFilterException ufe) {
+ fail(ufe.toString());
+ }
}
}
Modified: nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLNormalizers.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLNormalizers.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLNormalizers.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/net/TestURLNormalizers.java Sun May 19 21:11:54
2013
@@ -21,10 +21,12 @@ import java.net.MalformedURLException;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestURLNormalizers extends TestCase {
+public class TestURLNormalizers {
+ @Test
public void testURLNormalizers() {
Configuration conf = NutchConfiguration.create();
String clazz1 = "org.apache.nutch.net.urlnormalizer.regex.RegexURLNormalizer";
Modified: nutch/branches/2.x/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/parse/TestOutlinkExtractor.java Sun May 19
21:11:54 2013
@@ -22,7 +22,8 @@ import org.apache.nutch.parse.OutlinkExt
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/**
* TestCase to check regExp extraction of URLs.
@@ -31,7 +32,7 @@ import junit.framework.TestCase;
*
* @version 1.0
*/
-public class TestOutlinkExtractor extends TestCase {
+public class TestOutlinkExtractor {
private static Configuration conf = NutchConfiguration.create();
public void testGetNoOutlinks() {
@@ -46,6 +47,7 @@ public class TestOutlinkExtractor extend
assertEquals(0, outlinks.length);
}
+ @Test
public void testGetOutlinksHttp() {
Outlink[] outlinks = OutlinkExtractor.getOutlinks(
"Test with http://www.nutch.org/index.html is it found? " +
@@ -58,6 +60,7 @@ public class TestOutlinkExtractor extend
assertEquals("Wrong URL", "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl());
}
+ @Test
public void testGetOutlinksHttp2() {
Outlink[] outlinks = OutlinkExtractor.getOutlinks(
"Test with http://www.nutch.org/index.html is it found? " +
@@ -69,6 +72,8 @@ public class TestOutlinkExtractor extend
assertEquals("Wrong URL", "http://www.google.de", outlinks[1].getToUrl());
assertEquals("Wrong URL", "http://www.sybit.com/solutions/portals.html", outlinks[2].getToUrl());
}
+
+ @Test
public void testGetOutlinksFtp() {
Outlink[] outlinks = OutlinkExtractor.getOutlinks(
"Test with ftp://www.nutch.org is it found? " +
Modified: nutch/branches/2.x/src/test/org/apache/nutch/parse/TestParserFactory.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/parse/TestParserFactory.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/parse/TestParserFactory.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/parse/TestParserFactory.java Sun May 19 21:11:54
2013
@@ -17,8 +17,9 @@
package org.apache.nutch.parse;
-// JUnit imports
-import junit.framework.TestCase;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
// Nutch imports
import org.apache.nutch.plugin.Extension;
@@ -29,17 +30,15 @@ import org.apache.nutch.util.NutchConfig
* Unit test for new parse plugin selection.
*
* @author Sebastien Le Callonnec
- * @version 1.0
*/
-public class TestParserFactory extends TestCase {
+public class TestParserFactory {
private Configuration conf;
private ParserFactory parserFactory;
- public TestParserFactory(String name) { super(name); }
-
/** Inits the Test Case with the test parse-plugin file */
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
conf = NutchConfiguration.create();
conf.set("plugin.includes", ".*");
conf.set("parse.plugin.file",
@@ -48,6 +47,7 @@ public class TestParserFactory extends T
}
/** Unit test for <code>getExtensions(String)</code> method. */
+ @Test
public void testGetExtensions() throws Exception {
Extension ext = parserFactory.getExtensions("text/html").get(0);
assertEquals("parse-tika", ext.getDescriptor().getPluginId());
@@ -58,6 +58,7 @@ public class TestParserFactory extends T
}
/** Unit test to check <code>getParsers</code> method */
+ @Test
public void testGetParsers() throws Exception {
Parser [] parsers = parserFactory.getParsers("text/html", "http://foo.com");
assertNotNull(parsers);
Modified: nutch/branches/2.x/src/test/org/apache/nutch/plugin/TestPluginSystem.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/plugin/TestPluginSystem.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/plugin/TestPluginSystem.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/plugin/TestPluginSystem.java Sun May 19 21:11:54
2013
@@ -26,12 +26,14 @@ import java.util.LinkedList;
import java.util.Locale;
import java.util.Properties;
-import junit.framework.TestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobConf;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.NutchJobConf;
/**
@@ -39,14 +41,15 @@ import org.apache.nutch.util.NutchJobCon
*
* @author joa23
*/
-public class TestPluginSystem extends TestCase {
+public class TestPluginSystem {
private int fPluginCount;
- private LinkedList fFolders = new LinkedList();
+ private LinkedList<File> fFolders = new LinkedList<File>();
private Configuration conf ;
private PluginRepository repository;
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
this.conf = NutchConfiguration.create();
conf.set("plugin.includes", ".*");
// String string = this.conf.get("plugin.includes", "");
@@ -56,12 +59,8 @@ public class TestPluginSystem extends Te
this.repository = PluginRepository.get(conf);
}
- /*
- * (non-Javadoc)
- *
- * @see junit.framework.TestCase#tearDown()
- */
- protected void tearDown() throws Exception {
+ @After
+ public void tearDown() throws Exception {
for (int i = 0; i < fFolders.size(); i++) {
File folder = (File) fFolders.get(i);
delete(folder);
@@ -72,6 +71,7 @@ public class TestPluginSystem extends Te
/**
*/
+ @Test
public void testPluginConfiguration() {
String string = getPluginFolder();
File file = new File(string);
@@ -83,6 +83,7 @@ public class TestPluginSystem extends Te
/**
*/
+ @Test
public void testLoadPlugins() {
PluginDescriptor[] descriptors = repository
.getPluginDescriptors();
@@ -98,6 +99,7 @@ public class TestPluginSystem extends Te
}
}
+ @Test
public void testRepositoryCache() {
Configuration config = NutchConfiguration.create();
PluginRepository repo = PluginRepository.get(config);
@@ -117,6 +119,7 @@ public class TestPluginSystem extends Te
/**
*
*/
+ @Test
public void testGetExtensionAndAttributes() {
String xpId = " sdsdsd";
ExtensionPoint extensionPoint =repository
@@ -135,6 +138,7 @@ public class TestPluginSystem extends Te
/**
* @throws PluginRuntimeException
*/
+ @Test
public void testGetExtensionInstances() throws PluginRuntimeException {
Extension[] extensions = repository
.getExtensionPoint(getGetExtensionId()).getExtensions();
@@ -154,6 +158,7 @@ public class TestPluginSystem extends Te
*
*
*/
+ @Test
public void testGetClassLoader() {
PluginDescriptor[] descriptors = repository
.getPluginDescriptors();
@@ -166,6 +171,7 @@ public class TestPluginSystem extends Te
/**
* @throws IOException
*/
+ @Test
public void testGetResources() throws IOException {
PluginDescriptor[] descriptors = repository
.getPluginDescriptors();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestContent.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestContent.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestContent.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestContent.java Sun May 19 21:11:54
2013
@@ -24,17 +24,17 @@ import org.apache.nutch.util.NutchConfig
import org.apache.nutch.util.WritableTestUtils;
import org.apache.tika.mime.MimeTypes;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/** Unit tests for Content. */
-public class TestContent extends TestCase {
+public class TestContent {
private static Configuration conf = NutchConfiguration.create();
- public TestContent(String name) { super(name); }
-
+ @Test
public void testContent() throws Exception {
String page = "<HTML><BODY><H1>Hello World</H1><P>The Quick
Brown Fox Jumped Over the Lazy Fox.</BODY></HTML>";
@@ -55,6 +55,7 @@ public class TestContent extends TestCas
}
/** Unit tests for getContentType(String, String, byte[]) method. */
+ @Test
public void testGetContentType() throws Exception {
Content c = null;
Metadata p = new Metadata();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestProtocolFactory.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestProtocolFactory.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestProtocolFactory.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/protocol/TestProtocolFactory.java Sun May
19 21:11:54 2013
@@ -20,20 +20,24 @@ import org.apache.hadoop.conf.Configurat
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.ObjectCache;
-import junit.framework.TestCase;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestProtocolFactory extends TestCase {
+public class TestProtocolFactory {
Configuration conf;
ProtocolFactory factory;
- protected void setUp() throws Exception {
+ @Before
+ public void setUp() throws Exception {
conf = NutchConfiguration.create();
conf.set("plugin.includes", ".*");
conf.set("http.agent.name", "test-bot");
factory=new ProtocolFactory(conf);
}
+ @Test
public void testGetProtocol(){
//non existing protocol
@@ -69,6 +73,7 @@ public class TestProtocolFactory extends
}
}
+ @Test
public void testContains(){
assertTrue(factory.contains("http", "http"));
assertTrue(factory.contains("http", "http,ftp"));
Modified: nutch/branches/2.x/src/test/org/apache/nutch/storage/TestGoraStorage.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/storage/TestGoraStorage.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/storage/TestGoraStorage.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/storage/TestGoraStorage.java Sun May 19 21:11:54
2013
@@ -35,16 +35,34 @@ import org.apache.nutch.util.AbstractNut
import org.apache.nutch.util.CrawlTestUtil;
import org.hsqldb.Server;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
/**
* Tests basic Gora functionality by writing and reading webpages.
*/
public class TestGoraStorage extends AbstractNutchTest {
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
/**
* Sequentially read and write pages to a store.
*
* @throws Exception
*/
+ @Test
public void testSinglethreaded() throws Exception {
String id = "singlethread";
readWrite(id, webPageStore);
@@ -87,6 +105,7 @@ public class TestGoraStorage extends Abs
*
* @throws Exception
*/
+ @Test
public void testMultithreaded() throws Exception {
// create a fixed thread pool
int numThreads = 8;
@@ -127,6 +146,7 @@ public class TestGoraStorage extends Abs
*
* @throws Exception
*/
+ @Test
public void testMultiProcess() throws Exception {
// create and start a hsql server, a stand-alone (memory backed) db
// (important: a stand-alone server should be used because simple
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/AbstractNutchTest.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/AbstractNutchTest.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/AbstractNutchTest.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/AbstractNutchTest.java Sun May 19 21:11:54
2013
@@ -16,8 +16,6 @@
*/
package org.apache.nutch.util;
-import junit.framework.TestCase;
-
import org.apache.gora.store.DataStore;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -29,7 +27,7 @@ import org.apache.nutch.storage.WebPage;
* This class provides common routines for setup/teardown of an in-memory data
* store.
*/
-public class AbstractNutchTest extends TestCase {
+public class AbstractNutchTest {
protected Configuration conf;
protected FileSystem fs;
@@ -37,9 +35,7 @@ public class AbstractNutchTest extends T
protected DataStore<String, WebPage> webPageStore;
protected boolean persistentDataStore = false;
- @Override
public void setUp() throws Exception {
- super.setUp();
conf = CrawlTestUtil.createConfiguration();
conf.set("storage.data.store.class", "org.apache.gora.sql.store.SqlStore");
fs = FileSystem.get(conf);
@@ -47,7 +43,6 @@ public class AbstractNutchTest extends T
WebPage.class);
}
- @Override
public void tearDown() throws Exception {
// empty the database after test
if (!persistentDataStore) {
@@ -55,7 +50,6 @@ public class AbstractNutchTest extends T
webPageStore.flush();
webPageStore.close();
}
- super.tearDown();
fs.delete(testdir, true);
}
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestEncodingDetector.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestEncodingDetector.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestEncodingDetector.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestEncodingDetector.java Sun May 19
21:11:54 2013
@@ -19,14 +19,15 @@ package org.apache.nutch.util;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.storage.WebPage;
-public class TestEncodingDetector extends TestCase {
+public class TestEncodingDetector {
private static Configuration conf = NutchConfiguration.create();
private static byte[] contentInOctets;
@@ -39,10 +40,7 @@ public class TestEncodingDetector extend
}
}
- public TestEncodingDetector(String name) {
- super(name);
- }
-
+ @Test
public void testGuessing() {
// first disable auto detection
conf.setInt(EncodingDetector.MIN_CONFIDENCE_KEY, -1);
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestGZIPUtils.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestGZIPUtils.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestGZIPUtils.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestGZIPUtils.java Sun May 19 21:11:54
2013
@@ -17,15 +17,13 @@
package org.apache.nutch.util;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
import java.io.IOException;
/** Unit tests for GZIPUtils methods. */
-public class TestGZIPUtils extends TestCase {
- public TestGZIPUtils(String name) {
- super(name);
- }
+public class TestGZIPUtils {
/* a short, highly compressable, string */
String SHORT_TEST_STRING=
@@ -117,6 +115,7 @@ public class TestGZIPUtils extends TestC
// tests
+ @Test
public void testZipUnzip() {
byte[] testBytes= SHORT_TEST_STRING.getBytes();
testZipUnzip(testBytes);
@@ -126,6 +125,7 @@ public class TestGZIPUtils extends TestC
testZipUnzip(testBytes);
}
+ @Test
public void testZipUnzipBestEffort() {
byte[] testBytes= SHORT_TEST_STRING.getBytes();
testZipUnzipBestEffort(testBytes);
@@ -135,6 +135,7 @@ public class TestGZIPUtils extends TestC
testZipUnzipBestEffort(testBytes);
}
+ @Test
public void testTruncation() {
byte[] testBytes= SHORT_TEST_STRING.getBytes();
testTruncation(testBytes);
@@ -144,6 +145,7 @@ public class TestGZIPUtils extends TestC
testTruncation(testBytes);
}
+ @Test
public void testLimit() {
byte[] testBytes= SHORT_TEST_STRING.getBytes();
testLimit(testBytes);
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestNodeWalker.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestNodeWalker.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestNodeWalker.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestNodeWalker.java Sun May 19 21:11:54
2013
@@ -18,20 +18,17 @@
package org.apache.nutch.util;
import java.io.ByteArrayInputStream;
-import junit.framework.TestCase;
+
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
import org.apache.xerces.parsers.DOMParser;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
-
-
-
/** Unit tests for NodeWalker methods. */
-public class TestNodeWalker extends TestCase {
- public TestNodeWalker(String name) {
- super(name);
- }
+public class TestNodeWalker {
/* a snapshot of the nutch webpage */
private final static String WEBPAGE=
@@ -49,13 +46,15 @@ public class TestNodeWalker extends Test
private final static String[] ULCONTENT = new String[4];
- protected void setUp() throws Exception{
+ @Before
+ public void setUp() throws Exception{
ULCONTENT[0]="crawl several billion pages per month" ;
ULCONTENT[1]="maintain an index of these pages" ;
ULCONTENT[2]="search that index up to 1000 times per second" ;
ULCONTENT[3]="operate at minimal cost" ;
}
+ @Test
public void testSkipChildren() {
DOMParser parser= new DOMParser();
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestPrefixStringMatcher.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestPrefixStringMatcher.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestPrefixStringMatcher.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestPrefixStringMatcher.java Sun May
19 21:11:54 2013
@@ -17,13 +17,11 @@
package org.apache.nutch.util;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/** Unit tests for PrefixStringMatcher. */
-public class TestPrefixStringMatcher extends TestCase {
- public TestPrefixStringMatcher(String name) {
- super(name);
- }
+public class TestPrefixStringMatcher {
private final static int NUM_TEST_ROUNDS= 20;
private final static int MAX_TEST_PREFIXES= 100;
@@ -51,6 +49,7 @@ public class TestPrefixStringMatcher ext
return new String(chars);
}
+ @Test
public void testPrefixMatcher() {
int numMatches= 0;
int numInputsTested= 0;
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestStringUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestStringUtil.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestStringUtil.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestStringUtil.java Sun May 19 21:11:54
2013
@@ -17,14 +17,13 @@
package org.apache.nutch.util;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/** Unit tests for StringUtil methods. */
-public class TestStringUtil extends TestCase {
- public TestStringUtil(String name) {
- super(name);
- }
+public class TestStringUtil {
+ @Test
public void testRightPad() {
String s= "my string";
@@ -42,6 +41,7 @@ public class TestStringUtil extends Test
}
+ @Test
public void testLeftPad() {
String s= "my string";
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestSuffixStringMatcher.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestSuffixStringMatcher.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestSuffixStringMatcher.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestSuffixStringMatcher.java Sun May
19 21:11:54 2013
@@ -17,13 +17,11 @@
package org.apache.nutch.util;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/** Unit tests for SuffixStringMatcher. */
-public class TestSuffixStringMatcher extends TestCase {
- public TestSuffixStringMatcher(String name) {
- super(name);
- }
+public class TestSuffixStringMatcher {
private final static int NUM_TEST_ROUNDS= 20;
private final static int MAX_TEST_SUFFIXES= 100;
@@ -51,6 +49,7 @@ public class TestSuffixStringMatcher ext
return new String(chars);
}
+ @Test
public void testSuffixMatcher() {
int numMatches= 0;
int numInputsTested= 0;
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestTableUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestTableUtil.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestTableUtil.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestTableUtil.java Sun May 19 21:11:54
2013
@@ -17,9 +17,10 @@
package org.apache.nutch.util;
import org.apache.nutch.util.TableUtil;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
-public class TestTableUtil extends TestCase {
+public class TestTableUtil {
String urlString1 = "http://foo.com/";
String urlString2 = "http://foo.com:8900/";
@@ -38,6 +39,7 @@ public class TestTableUtil extends TestC
String reversedUrlString6 = "com.foo:http";
String reversedUrlString7 = ":file/var/www/index.html";
+ @Test
public void testReverseUrl() throws Exception {
assertReverse(urlString1, reversedUrlString1);
assertReverse(urlString2, reversedUrlString2);
@@ -49,6 +51,7 @@ public class TestTableUtil extends TestC
assertReverse(urlString7, reversedUrlString7);
}
+ @Test
public void testUnreverseUrl() throws Exception {
assertUnreverse(reversedUrlString1, urlString1);
assertUnreverse(reversedUrlString2, urlString2);
Modified: nutch/branches/2.x/src/test/org/apache/nutch/util/TestURLUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/util/TestURLUtil.java?rev=1484348&r1=1484347&r2=1484348&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/util/TestURLUtil.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/util/TestURLUtil.java Sun May 19 21:11:54
2013
@@ -19,18 +19,13 @@ package org.apache.nutch.util;
import java.net.URL;
-import junit.framework.TestCase;
+import org.junit.Test;
+import static org.junit.Assert.*;
/** Test class for URLUtil */
-public class TestURLUtil
- extends TestCase {
-
- @Override
- protected void setUp()
- throws Exception {
- super.setUp();
- }
+public class TestURLUtil {
+ @Test
public void testGetDomainName()
throws Exception {
@@ -81,6 +76,7 @@ public class TestURLUtil
}
+ @Test
public void testGetDomainSuffix()
throws Exception {
URL url = null;
@@ -133,6 +129,7 @@ public class TestURLUtil
}
+ @Test
public void testGetHostBatches()
throws Exception {
URL url;
@@ -165,6 +162,7 @@ public class TestURLUtil
}
+ @Test
public void testChooseRepr()
throws Exception {
Author: tejasp
Date: Thu May 9 07:49:40 2013
New Revision: 1480551
URL: http://svn.apache.org/r1480551
Log:
Added crawler-commons dependency in pom.xml
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/default.properties
nutch/trunk/pom.xml
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1480551&r1=1480550&r2=1480551&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu May 9 07:49:40 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* Added crawler-commons dependency in pom.xml (tejasp)
+
* NUTCH-956 solrindex issues: add field tld to Solr schema (Alexis via lewismc, snagel)
* NUTCH-1277 Fix [fallthrough] javac warnings (tejasp)
@@ -34,7 +36,7 @@ Nutch Change Log
* NUTCH-1521 CrawlDbFilter pass null url to urlNormalizers (Lufeng via lewismc)
-* NUTCH-1284 Add site fetcher.max.crawl.delay as log output by default (Tejas Patil)
+* NUTCH-1284 Add site fetcher.max.crawl.delay as log output by default (tejasp)
* NUTCH-1453 Substantiate tests for IndexingFilters (lufeng via lewismc)
@@ -48,17 +50,17 @@ Nutch Change Log
* NUTCH-1500 bin/crawl fails on step solrindex with wrong path to segment (Tristan Buckner,
snagel)
-* NUTCH-1274 Fix [cast] javac warnings (Tejas Patil via lewismc)
+* NUTCH-1274 Fix [cast] javac warnings (tejasp via lewismc)
-* NUTCH-1494 RSS feed plugin seems broken (Sourajit Basak, Tejas Patil and lewismc)
+* NUTCH-1494 RSS feed plugin seems broken (Sourajit Basak, tejasp and lewismc)
-* NUTCH-1127 JUnit test for urlfilter-validator (Tejas Patil via lewismc)
+* NUTCH-1127 JUnit test for urlfilter-validator (tejasp via lewismc)
-* NUTCH-1119 JUnit test for index-static (Tejas Patil via lewismc)
+* NUTCH-1119 JUnit test for index-static (tejasp via lewismc)
* NUTCH-1510 Upgrade to Hadoop 1.1.1 (markus)
-* NUTCH-1118 JUnit test for index-basic (Tejas Patil via lewismc)
+* NUTCH-1118 JUnit test for index-basic (tejasp via lewismc)
* NUTCH-1331 limit crawler to defined depth (jnioche)
Modified: nutch/trunk/default.properties
URL: http://svn.apache.org/viewvc/nutch/trunk/default.properties?rev=1480551&r1=1480550&r2=1480551&view=diff
==============================================================================
--- nutch/trunk/default.properties (original)
+++ nutch/trunk/default.properties Thu May 9 07:49:40 2013
@@ -16,7 +16,7 @@
name=apache-nutch
version=1.7-SNAPSHOT
final.name=${name}-${version}
-year=2012
+year=2013
basedir = ./
src.dir = ./src/java
Modified: nutch/trunk/pom.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/pom.xml?rev=1480551&r1=1480550&r2=1480551&view=diff
==============================================================================
--- nutch/trunk/pom.xml (original)
+++ nutch/trunk/pom.xml Thu May 9 07:49:40 2013
@@ -93,6 +93,21 @@
<name>Ferdy Galema</name>
<email>ferdy@apache.org</email>
</developer>
+ <developer>
+ <id>tejasp</id>
+ <name>Tejas Patil</name>
+ <email>tejasp@apache.org</email>
+ </developer>
+ <developer>
+ <id>kiranch</id>
+ <name>Kiran Chitturi</name>
+ <email>kiranch@apache.org</email>
+ </developer>
+ <developer>
+ <id>fenglu</id>
+ <name>Feng</name>
+ <email>fenglu@apache.org</email>
+ </developer>
</developers>
<build>
<testSourceDirectory>${basedir}/src/test</testSourceDirectory>
@@ -295,6 +310,12 @@
<version>6.1.22</version>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>com.google.code.crawler-commons</groupId>
+ <artifactId>crawler-commons</artifactId>
+ <version>0.2</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
Author: tejasp
Date: Thu May 9 07:45:09 2013
New Revision: 1480550
URL: http://svn.apache.org/r1480550
Log:
Added crawler-commons dependency in pom.xml
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/default.properties
nutch/branches/2.x/pom.xml
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1480550&r1=1480549&r2=1480550&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Thu May 9 07:45:09 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* Added crawler-commons dependency in pom.xml (tejasp)
+
* NUTCH-956 solrindex issues: add field tld to Solr schema (Alexis via lewismc, snagel)
* NUTCH-1277 Fix [fallthrough] javac warnings (tejasp)
@@ -46,11 +48,11 @@ Release 2.2 - Current Development
* NUTCH-XX remove unused db.max.inlinks property in nutch-default.xml (lewismc)
-* NUTCH-1284 Add site fetcher.max.crawl.delay as log output by default (Tejas Patil)
+* NUTCH-1284 Add site fetcher.max.crawl.delay as log output by default (tejasp)
* NUTCH-1453 Substantiate tests for IndexingFilters (lufeng via lewismc)
-* NUTCH-1274 Fix [cast] javac warnings (Tejas Patil via lewismc)
+* NUTCH-1274 Fix [cast] javac warnings (tejasp via lewismc)
* NUTCH-1516 Nutch 2.x pom.xml out of sync with ivy.xml (lewismc)
Modified: nutch/branches/2.x/default.properties
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/default.properties?rev=1480550&r1=1480549&r2=1480550&view=diff
==============================================================================
--- nutch/branches/2.x/default.properties (original)
+++ nutch/branches/2.x/default.properties Thu May 9 07:45:09 2013
@@ -17,7 +17,7 @@
name=apache-nutch
version=2.2-SNAPSHOT
final.name=${name}-${version}
-year=2012
+year=2013
basedir = ./
src.dir = ./src/java
Modified: nutch/branches/2.x/pom.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/pom.xml?rev=1480550&r1=1480549&r2=1480550&view=diff
==============================================================================
--- nutch/branches/2.x/pom.xml (original)
+++ nutch/branches/2.x/pom.xml Thu May 9 07:45:09 2013
@@ -82,6 +82,21 @@
<name>Lewis John McGibbney</name>
<email>lewismc@apache.org</email>
</developer>
+ <developer>
+ <id>tejasp</id>
+ <name>Tejas Patil</name>
+ <email>tejasp@apache.org</email>
+ </developer>
+ <developer>
+ <id>kiranch</id>
+ <name>Kiran Chitturi</name>
+ <email>kiranch@apache.org</email>
+ </developer>
+ <developer>
+ <id>fenglu</id>
+ <name>Feng</name>
+ <email>fenglu@apache.org</email>
+ </developer>
</developers>
<build>
<sourceDirectory>src/java</sourceDirectory>
@@ -247,12 +262,6 @@
<optional>true</optional>
</dependency>
<dependency>
- <groupId>org.jdom</groupId>
- <artifactId>jdom</artifactId>
- <version>1.1</version>
- <optional>true</optional>
- </dependency>
- <dependency>
<groupId>org.apache.gora</groupId>
<artifactId>gora-sql</artifactId>
<version>0.1.1-incubating</version>
@@ -270,6 +279,12 @@
<version>2.0.5</version>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>com.google.code.crawler-commons</groupId>
+ <artifactId>crawler-commons</artifactId>
+ <version>0.2</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>
Author: snagel
Date: Wed May 8 22:04:53 2013
New Revision: 1480485
URL: http://svn.apache.org/r1480485
Log:
NUTCH-956 solrindex issues
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/schema-solr4.xml
nutch/trunk/conf/schema.xml
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1480485&r1=1480484&r2=1480485&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Wed May 8 22:04:53 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-956 solrindex issues: add field tld to Solr schema (Alexis via lewismc, snagel)
+
* NUTCH-1277 Fix [fallthrough] javac warnings (tejasp)
* NUTCH-1514 Phase out the deprecated configuration properties (if possible) (tejasp)
Modified: nutch/trunk/conf/schema-solr4.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/schema-solr4.xml?rev=1480485&r1=1480484&r2=1480485&view=diff
==============================================================================
--- nutch/trunk/conf/schema-solr4.xml (original)
+++ nutch/trunk/conf/schema-solr4.xml Wed May 8 22:04:53 2013
@@ -345,6 +345,9 @@
<!-- fields for creativecommons plugin -->
<field name="cc" type="string" stored="true" indexed="true" multiValued="true"/>
+
+ <!-- fields for tld plugin -->
+ <field name="tld" type="string" stored="false" indexed="false"/>
</fields>
<uniqueKey>id</uniqueKey>
<defaultSearchField>text</defaultSearchField>
Modified: nutch/trunk/conf/schema.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/schema.xml?rev=1480485&r1=1480484&r2=1480485&view=diff
==============================================================================
--- nutch/trunk/conf/schema.xml (original)
+++ nutch/trunk/conf/schema.xml Wed May 8 22:04:53 2013
@@ -114,6 +114,9 @@
<!-- fields for creativecommons plugin -->
<field name="cc" type="string" stored="true" indexed="true"
multiValued="true"/>
+
+ <!-- fields for tld plugin -->
+ <field name="tld" type="string" stored="false" indexed="false"/>
</fields>
<uniqueKey>id</uniqueKey>
<defaultSearchField>content</defaultSearchField>
Modified: nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=1480485&r1=1480484&r2=1480485&view=diff
==============================================================================
--- nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
(original)
+++ nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Wed May 8 22:04:53 2013
@@ -52,12 +52,13 @@ import org.apache.commons.lang.StringUti
import org.apache.commons.lang.time.DateUtils;
/**
- * Add (or reset) a few metaData properties as respective fields
- * (if they are available), so that they can be displayed by more.jsp
- * (called by search.jsp).
- *
- * content-type is indexed to support query by type:
- * last-modifed is indexed to support query by date:
+ * Add (or reset) a few metaData properties as respective fields (if they are
+ * available), so that they can be accurately used within the search index.
+ *
+ * 'lastModifed' is indexed to support query by date, 'contentLength' obtains content length
from the HTTP
+ * header, 'type' field is indexed to support query by type and finally the 'title' field
is an attempt
+ * to reset the title if a content-disposition hint exists. The logic is that such a presence
is indicative
+ * that the content provider wants the filename therein to be used as the title.
*
* Still need to make content-length searchable!
*
Author: snagel
Date: Wed May 8 22:04:04 2013
New Revision: 1480484
URL: http://svn.apache.org/r1480484
Log:
NUTCH-956 solrindex issues
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/conf/schema-solr4.xml
nutch/branches/2.x/conf/schema.xml
nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1480484&r1=1480483&r2=1480484&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Wed May 8 22:04:04 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-956 solrindex issues: add field tld to Solr schema (Alexis via lewismc, snagel)
+
* NUTCH-1277 Fix [fallthrough] javac warnings (tejasp)
* NUTCH-1514 Phase out the deprecated configuration properties (if possible) (tejasp)
Modified: nutch/branches/2.x/conf/schema-solr4.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/conf/schema-solr4.xml?rev=1480484&r1=1480483&r2=1480484&view=diff
==============================================================================
--- nutch/branches/2.x/conf/schema-solr4.xml (original)
+++ nutch/branches/2.x/conf/schema-solr4.xml Wed May 8 22:04:04 2013
@@ -346,6 +346,9 @@
<!-- fields for creativecommons plugin -->
<field name="cc" type="string" stored="true" indexed="true" multiValued="true"/>
+
+ <!-- fields for tld plugin -->
+ <field name="tld" type="string" stored="false" indexed="false"/>
</fields>
<uniqueKey>id</uniqueKey>
<defaultSearchField>text</defaultSearchField>
Modified: nutch/branches/2.x/conf/schema.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/conf/schema.xml?rev=1480484&r1=1480483&r2=1480484&view=diff
==============================================================================
--- nutch/branches/2.x/conf/schema.xml (original)
+++ nutch/branches/2.x/conf/schema.xml Wed May 8 22:04:04 2013
@@ -114,6 +114,9 @@
<!-- fields for creativecommons plugin -->
<field name="cc" type="string" stored="true" indexed="true"
multiValued="true"/>
+
+ <!-- fields for tld plugin -->
+ <field name="tld" type="string" stored="false" indexed="false"/>
</fields>
<uniqueKey>id</uniqueKey>
<defaultSearchField>content</defaultSearchField>
Modified: nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=1480484&r1=1480483&r2=1480484&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
(original)
+++ nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Wed May 8 22:04:04 2013
@@ -44,10 +44,12 @@ import org.slf4j.LoggerFactory;
/**
* Add (or reset) a few metaData properties as respective fields (if they are
- * available), so that they can be displayed by more.jsp (called by search.jsp).
+ * available), so that they can be accurately used within the search index.
*
- * content-type is indexed to support query by type: last-modifed is indexed to
- * support query by date:
+ * 'lastModifed' is indexed to support query by date, 'contentLength' obtains content length
from the HTTP
+ * header, 'type' field is indexed to support query by type and finally the 'title' field
is an attempt
+ * to reset the title if a content-disposition hint exists. The logic is that such a presence
is indicative
+ * that the content provider wants the filename therein to be used as the title.
*
* Still need to make content-length searchable!
*
@@ -171,7 +173,9 @@ public class MoreIndexingFilter implemen
*/
private NutchDocument addType(NutchDocument doc, WebPage page, String url) {
String mimeType = null;
- Utf8 contentType = page.getFromHeaders(new Utf8(HttpHeaders.CONTENT_TYPE));
+ Utf8 contentType = page.getContentType();
+ if (contentType == null)
+ contentType = page.getFromHeaders(new Utf8(HttpHeaders.CONTENT_TYPE));
if (contentType == null) {
// Note by Jerome Charron on 20050415:
// Content Type not solved by a previous plugin
@@ -194,13 +198,11 @@ public class MoreIndexingFilter implemen
return doc;
}
- //String scontentType = mimeType.getName();
-
doc.add("type", mimeType);
// Check if we need to split the content type in sub parts
- if ( null != contentType && conf.getBoolean("moreIndexingFilter.indexMimeTypeParts",
true)) {
- String[] parts = getParts(contentType.toString());
+ if (conf.getBoolean("moreIndexingFilter.indexMimeTypeParts", true)) {
+ String[] parts = getParts(mimeType);
for(String part: parts) {
doc.add("type", part);
Author: tejasp
Date: Tue May 7 06:42:14 2013
New Revision: 1479775
URL: http://svn.apache.org/r1479775
Log:
NUTCH-1277 Fix [fallthrough] javac warnings
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1479775&r1=1479774&r2=1479775&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue May 7 06:42:14 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1277 Fix [fallthrough] javac warnings (tejasp)
+
* NUTCH-1514 Phase out the deprecated configuration properties (if possible) (tejasp)
* NUTCH-1334 NPE in FetcherOutputFormat (jnioche via tejasp)
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=1479775&r1=1479774&r2=1479775&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java Tue May 7 06:42:14 2013
@@ -627,6 +627,7 @@ public class Fetcher extends Configured
outlinksDepthDivisor = conf.getInt("fetcher.follow.outlinks.depth.divisor", 2);
}
+ @SuppressWarnings("fallthrough")
public void run() {
activeThreads.incrementAndGet(); // count threads
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java?rev=1479775&r1=1479774&r2=1479775&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/OldFetcher.java Tue May 7 06:42:14 2013
@@ -111,6 +111,7 @@ public class OldFetcher extends Configur
this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER);
}
+ @SuppressWarnings("fallthrough")
public void run() {
synchronized (OldFetcher.this) {activeThreads++;} // count threads
Author: tejasp
Date: Tue May 7 06:41:34 2013
New Revision: 1479774
URL: http://svn.apache.org/r1479774
Log:
NUTCH-1277 Fix [fallthrough] javac warnings
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java
nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1479774&r1=1479773&r2=1479774&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Tue May 7 06:41:34 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1277 Fix [fallthrough] javac warnings (tejasp)
+
* NUTCH-1514 Phase out the deprecated configuration properties (if possible) (tejasp)
* NUTCH-1273 Fix [deprecation] javac warnings (lewsimc + tejasp)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java?rev=1479774&r1=1479773&r2=1479774&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/api/impl/RAMJobManager.java Tue May 7 06:41:34
2013
@@ -110,6 +110,7 @@ public class RAMJobManager implements Jo
}
@Override
+ @SuppressWarnings("fallthrough")
public List<JobStatus> list(String crawlId, State state) throws Exception {
List<JobStatus> res = new ArrayList<JobStatus>();
if (state == null) state = State.ANY;
Modified: nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java?rev=1479774&r1=1479773&r2=1479774&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java Tue May 7 06:41:34
2013
@@ -451,6 +451,7 @@ extends GoraReducer<IntWritable, FetchEn
}
@Override
+ @SuppressWarnings("fallthrough")
public void run() {
activeThreads.incrementAndGet(); // count threads
Author: tejasp
Date: Fri May 3 19:27:39 2013
New Revision: 1478939
URL: http://svn.apache.org/r1478939
Log:
NUTCH-1514 Phase out the deprecated configuration properties (if possible)
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java
nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1478939&r1=1478938&r2=1478939&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Fri May 3 19:27:39 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1514 Phase out the deprecated configuration properties (if possible) (tejasp)
+
* NUTCH-1334 NPE in FetcherOutputFormat (jnioche via tejasp)
* NUTCH-1549 Fix deprecated use of Tika MimeType API in o.a.n.util.MimeUtil (tejasp)
Modified: nutch/trunk/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1478939&r1=1478938&r2=1478939&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Fri May 3 19:27:39 2013
@@ -325,13 +325,6 @@
<!-- web db properties -->
<property>
- <name>db.default.fetch.interval</name>
- <value>30</value>
- <description>(DEPRECATED) The default number of days between re-fetches of a page.
- </description>
-</property>
-
-<property>
<name>db.fetch.interval.default</name>
<value>2592000</value>
<description>The default number of seconds between re-fetches of a page (30 days).
@@ -611,14 +604,6 @@
</property>
<property>
- <name>generate.max.per.host</name>
- <value>-1</value>
- <description>(Deprecated). Use generate.max.count and generate.count.mode instead.
- The maximum number of urls per host in a single
- fetchlist. -1 if unlimited.</description>
-</property>
-
-<property>
<name>generate.min.score</name>
<value>0</value>
<description>Select only entries with a score larger than
@@ -698,8 +683,7 @@
<name>fetcher.threads.per.queue</name>
<value>1</value>
<description>This number is the maximum number of threads that
- should be allowed to access a queue at one time. Replaces
- deprecated parameter 'fetcher.threads.per.host'.
+ should be allowed to access a queue at one time.
</description>
</property>
@@ -707,8 +691,7 @@
<name>fetcher.queue.mode</name>
<value>byHost</value>
<description>Determines how to put URLs into queues. Default value is 'byHost',
- also takes 'byDomain' or 'byIP'. Replaces the deprecated parameter
- 'fetcher.threads.per.host.by.ip'.
+ also takes 'byDomain' or 'byIP'.
</description>
</property>
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java?rev=1478939&r1=1478938&r2=1478939&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/AbstractFetchSchedule.java Fri May 3 19:27:39
2013
@@ -47,12 +47,8 @@ public abstract class AbstractFetchSched
public void setConf(Configuration conf) {
super.setConf(conf);
if (conf == null) return;
- int oldDefaultInterval = conf.getInt("db.default.fetch.interval", 0);
defaultInterval = conf.getInt("db.fetch.interval.default", 0);
- if (oldDefaultInterval > 0 && defaultInterval == 0) defaultInterval = oldDefaultInterval
* SECONDS_PER_DAY;
- int oldMaxInterval = conf.getInt("db.max.fetch.interval", 0);
maxInterval = conf.getInt("db.fetch.interval.max", 0 );
- if (oldMaxInterval > 0 && maxInterval == 0) maxInterval = oldMaxInterval *
FetchSchedule.SECONDS_PER_DAY;
LOG.info("defaultInterval=" + defaultInterval);
LOG.info("maxInterval=" + maxInterval);
}
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java?rev=1478939&r1=1478938&r2=1478939&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java Fri May 3 19:27:39 2013
@@ -76,7 +76,6 @@ public class Generator extends Configure
// deprecated parameters
public static final String GENERATE_MAX_PER_HOST_BY_IP = "generate.max.per.host.by.ip";
- public static final String GENERATE_MAX_PER_HOST = "generate.max.per.host";
public static class SelectorEntry implements Writable {
public Text url;
@@ -140,10 +139,7 @@ public class Generator extends Configure
curTime = job.getLong(GENERATOR_CUR_TIME, System.currentTimeMillis());
limit = job.getLong(GENERATOR_TOP_N, Long.MAX_VALUE) / job.getNumReduceTasks();
maxCount = job.getInt(GENERATOR_MAX_COUNT, -1);
- // back compatibility with old param
- int oldMaxPerHost = job.getInt(GENERATE_MAX_PER_HOST, -1);
- if (maxCount==-1 && oldMaxPerHost!=-1){
- maxCount = oldMaxPerHost;
+ if (maxCount==-1){
byDomain = false;
}
if (GENERATOR_COUNT_VALUE_DOMAIN.equals(job.get(GENERATOR_COUNT_MODE))) byDomain =
true;
Author: tejasp
Date: Fri May 3 19:27:06 2013
New Revision: 1478938
URL: http://svn.apache.org/r1478938
Log:
NUTCH-1514 Phase out the deprecated configuration properties (if possible)
Modified:
nutch/branches/2.x/conf/nutch-default.xml
Modified: nutch/branches/2.x/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/conf/nutch-default.xml?rev=1478938&r1=1478937&r2=1478938&view=diff
==============================================================================
--- nutch/branches/2.x/conf/nutch-default.xml (original)
+++ nutch/branches/2.x/conf/nutch-default.xml Fri May 3 19:27:06 2013
@@ -318,13 +318,6 @@
<!-- web db properties -->
<property>
- <name>db.default.fetch.interval</name>
- <value>30</value>
- <description>(DEPRECATED) The default number of days between re-fetches of a page.
- </description>
-</property>
-
-<property>
<name>db.fetch.interval.default</name>
<value>2592000</value>
<description>The default number of seconds between re-fetches of a page (30 days).
Author: tejasp
Date: Fri May 3 19:24:41 2013
New Revision: 1478937
URL: http://svn.apache.org/r1478937
Log:
NUTCH-1514 Phase out the deprecated configuration properties (if possible)
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java
nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1478937&r1=1478936&r2=1478937&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Fri May 3 19:24:41 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1514 Phase out the deprecated configuration properties (if possible) (tejasp)
+
* NUTCH-1273 Fix [deprecation] javac warnings (lewsimc + tejasp)
* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java?rev=1478937&r1=1478936&r2=1478937&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/GeneratorJob.java Fri May 3 19:24:41
2013
@@ -236,7 +236,7 @@ public class GeneratorJob extends NutchT
System.out.println(" -noFilter - do not activate the filter plugin to filter
the url, default is true ");
System.out.println(" -noNorm - do not activate the normalizer plugin to normalize
the url, default is true ");
System.out.println(" -adddays - Adds numDays to the current time to facilitate
crawling urls already");
- System.out.println(" fetched sooner then db.default.fetch.interval.
Default value is 0.");
+ System.out.println(" fetched sooner then db.fetch.interval.default.
Default value is 0.");
System.out.println("----------------------");
System.out.println("Please set the params.");
return -1;
Modified: nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java?rev=1478937&r1=1478936&r2=1478937&view=diff
==============================================================================
--- nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java (original)
+++ nutch/branches/2.x/src/test/org/apache/nutch/crawl/TestGenerator.java Fri May 3 19:24:41
2013
@@ -107,7 +107,7 @@ public class TestGenerator extends Abstr
}
/**
- * Test that generator obeys the property "generate.max.per.host".
+ * Test that generator obeys the property "generate.max.count" and "generate.count.mode".
*
* @throws Exception
*/
Author: tejasp
Date: Tue Apr 30 21:42:19 2013
New Revision: 1477847
URL: http://svn.apache.org/r1477847
Log:
NUTCH-1334 NPE in FetcherOutputFormat
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1477847&r1=1477846&r2=1477847&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Apr 30 21:42:19 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1334 NPE in FetcherOutputFormat (jnioche via tejasp)
+
* NUTCH-1549 Fix deprecated use of Tika MimeType API in o.a.n.util.MimeUtil (tejasp)
* NUTCH-346 Improve readability of logs/hadoop.log (Renaud Richardet via tejasp)
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java?rev=1477847&r1=1477846&r2=1477847&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java Tue Apr 30 21:42:19
2013
@@ -98,9 +98,9 @@ public class FetcherOutputFormat impleme
if (w instanceof CrawlDatum)
fetchOut.append(key, w);
- else if (w instanceof Content)
+ else if (w instanceof Content && contentOut != null)
contentOut.append(key, w);
- else if (w instanceof Parse)
+ else if (w instanceof Parse && parseOut != null)
parseOut.write(key, (Parse)w);
}
Author: tejasp
Date: Tue Apr 30 20:51:44 2013
New Revision: 1477821
URL: http://svn.apache.org/r1477821
Log:
NUTCH-1549 Fix deprecated use of Tika MimeType API in o.a.n.util.MimeUtil
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/util/MimeUtil.java
nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1477821&r1=1477820&r2=1477821&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Apr 30 20:51:44 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1549 Fix deprecated use of Tika MimeType API in o.a.n.util.MimeUtil (tejasp)
+
* NUTCH-346 Improve readability of logs/hadoop.log (Renaud Richardet via tejasp)
* NUTCH-829 duplicate hadoop temp files (Mike Baranczak, lewismc, tejasp)
Modified: nutch/trunk/src/java/org/apache/nutch/util/MimeUtil.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/util/MimeUtil.java?rev=1477821&r1=1477820&r2=1477821&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/util/MimeUtil.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/util/MimeUtil.java Tue Apr 30 20:51:44 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
// Tika imports
import org.apache.tika.Tika;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.mime.MimeType;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
@@ -169,12 +170,19 @@ public final class MimeUtil {
|| (type != null && type.getName().equals(MimeTypes.OCTET_STREAM))) {
// If no mime-type header, or cannot find a corresponding registered
// mime-type, then guess a mime-type from the url pattern
- type = this.mimeTypes.getMimeType(url) != null ? this.mimeTypes
- .getMimeType(url) : type;
+ try {
+ TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
+ Tika tika = new Tika(tikaConfig);
+ retType = tika.detect(url) != null ? tika.detect(url) : null;
+ } catch (Exception e) {
+ String message = "Problem loading default Tika configuration";
+ LOG.error(message, e);
+ throw new RuntimeException(e);
+ }
+ } else {
+ retType = type.getName();
}
- retType= type.getName();
-
// if magic is enabled use mime magic to guess if the mime type returned
// from the magic guess is different than the one that's already set so far
// if it is, and it's not the default mime type, then go with the mime type
Modified: nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=1477821&r1=1477820&r2=1477821&view=diff
==============================================================================
--- nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
(original)
+++ nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Tue Apr 30 20:51:44 2013
@@ -16,8 +16,6 @@
*/
package org.apache.nutch.indexer.more;
-import org.apache.tika.mime.MimeType;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,19 +34,16 @@ import org.apache.nutch.crawl.CrawlDatum
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.util.MimeUtil;
+import org.apache.tika.Tika;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import java.text.ParseException;
-import java.text.SimpleDateFormat;
import java.io.BufferedReader;
-import java.io.FileReader;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.Date;
import java.util.regex.*;
import java.util.HashMap;
@@ -72,11 +67,9 @@ import org.apache.commons.lang.time.Date
public class MoreIndexingFilter implements IndexingFilter {
public static final Logger LOG = LoggerFactory.getLogger(MoreIndexingFilter.class);
- /** A flag that tells if magic resolution must be performed */
- private boolean MAGIC;
-
/** Get the MimeTypes resolver instance. */
private MimeUtil MIME;
+ private Tika tika = new Tika();
/** Map for mime-type substitution */
private HashMap<String,String> mimeMap = null;
@@ -114,7 +107,6 @@ public class MoreIndexingFilter implemen
// un-stored, indexed and un-tokenized
doc.add("date", new Date(time));
-
return doc;
}
@@ -220,7 +212,8 @@ public class MoreIndexingFilter implemen
// } else {
// contentType = MIME.getMimeType(url);
// }
- mimeType = MIME.getMimeType(url);
+
+ mimeType = tika.detect(url);
} else {
mimeType = MIME.forName(MimeUtil.cleanMimeType(contentType));
}
@@ -240,7 +233,6 @@ public class MoreIndexingFilter implemen
}
contentType = mimeType;
-
doc.add("type", contentType);
// Check if we need to split the content type in sub parts
Modified: nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java?rev=1477821&r1=1477820&r2=1477821&view=diff
==============================================================================
--- nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java
(original)
+++ nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java
Tue Apr 30 20:51:44 2013
@@ -38,13 +38,10 @@ import org.apache.nutch.net.protocols.Re
import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.parse.ParseUtil;
-import org.apache.nutch.parse.ParseImpl;
import org.apache.nutch.parse.ParseException;
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.util.MimeUtil;
-
-
+import org.apache.tika.Tika;
/**
*
@@ -52,26 +49,18 @@ import org.apache.nutch.util.MimeUtil;
*/
public class ZipTextExtractor {
- /** Get the MimeTypes resolver instance. */
- private MimeUtil MIME;
-
public static final Logger LOG = LoggerFactory.getLogger(ZipTextExtractor.class);
private Configuration conf;
-
-
+
/** Creates a new instance of ZipTextExtractor */
public ZipTextExtractor(Configuration conf) {
this.conf = conf;
- this.MIME = new MimeUtil(conf);
}
- public String extractText(InputStream input, String url, List outLinksList) throws IOException
{
+ public String extractText(InputStream input, String url, List<Outlink> outLinksList)
throws IOException {
String resultText = "";
- byte temp;
-
ZipInputStream zin = new ZipInputStream(input);
-
ZipEntry entry;
while ((entry = zin.getNextEntry()) != null) {
@@ -93,7 +82,8 @@ public class ZipTextExtractor {
int i = fname.lastIndexOf('.');
if (i != -1) {
// Trying to resolve the Mime-Type
- String contentType = MIME.getMimeType(fname);
+ Tika tika = new Tika();
+ String contentType = tika.detect(fname);
try {
Metadata metadata = new Metadata();
metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize()));
Modified: nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=1477821&r1=1477820&r2=1477821&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
(original)
+++ nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Tue Apr 30 20:51:44 2013
@@ -224,7 +224,7 @@ public class FileResponse {
headers.set(Response.LAST_MODIFIED,
HttpDateFormat.toString(f.lastModified()));
- String mimeType = MIME.getMimeType(f);
+ String mimeType = tika.detect(f);
headers.set(Response.CONTENT_TYPE, mimeType != null ? mimeType : "");
Author: tejasp
Date: Tue Apr 30 19:36:23 2013
New Revision: 1477792
URL: http://svn.apache.org/r1477792
Log:
NUTCH-1273 Fix [deprecation] javac warnings
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/ivy/ivy.xml
nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginDescriptor.java
nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginManifestParser.java
nutch/branches/2.x/src/java/org/apache/nutch/util/MimeUtil.java
nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Tue Apr 30 19:36:23 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1273 Fix [deprecation] javac warnings (lewsimc + tejasp)
+
* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
* NUTCH-346 Improve readability of logs/hadoop.log (Renaud Richardet via tejasp)
Modified: nutch/branches/2.x/ivy/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/ivy/ivy.xml?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/ivy/ivy.xml (original)
+++ nutch/branches/2.x/ivy/ivy.xml Tue Apr 30 19:36:23 2013
@@ -65,6 +65,7 @@
<dependency org="xerces" name="xercesImpl" rev="2.9.1" />
<dependency org="xerces" name="xmlParserAPIs" rev="2.6.2" />
+ <dependency org="xalan" name="serializer" rev="2.7.1" />
<dependency org="oro" name="oro" rev="2.0.8" />
<dependency org="org.jdom" name="jdom" rev="1.1" conf="*->default" />
Modified: nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginDescriptor.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginDescriptor.java?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginDescriptor.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginDescriptor.java Tue Apr 30 19:36:23
2013
@@ -219,7 +219,7 @@ public class PluginDescriptor {
*/
public void addExportedLibRelative(String pLibPath)
throws MalformedURLException {
- URL url = new File(getPluginPath() + File.separator + pLibPath).toURL();
+ URL url = new File(getPluginPath() + File.separator + pLibPath).toURI().toURL();
fExportedLibs.add(url);
}
@@ -248,7 +248,7 @@ public class PluginDescriptor {
*/
public void addNotExportedLibRelative(String pLibPath)
throws MalformedURLException {
- URL url = new File(getPluginPath() + File.separator + pLibPath).toURL();
+ URL url = new File(getPluginPath() + File.separator + pLibPath).toURI().toURL();
fNotExportedLibs.add(url);
}
@@ -279,7 +279,7 @@ public class PluginDescriptor {
try {
for (File file2 : file.listFiles()) {
if (file2.getAbsolutePath().endsWith("properties"))
- arrayList.add(file2.getParentFile().toURL());
+ arrayList.add(file2.getParentFile().toURI().toURL());
}
} catch (MalformedURLException e) {
LOG.debug(getPluginId() + " " + e.toString());
Modified: nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginManifestParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginManifestParser.java?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginManifestParser.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/plugin/PluginManifestParser.java Tue Apr
30 19:36:23 2013
@@ -147,7 +147,7 @@ public class PluginManifestParser {
private PluginDescriptor parseManifestFile(String pManifestPath)
throws MalformedURLException, SAXException, IOException,
ParserConfigurationException {
- Document document = parseXML(new File(pManifestPath).toURL());
+ Document document = parseXML(new File(pManifestPath).toURI().toURL());
String pPath = new File(pManifestPath).getParent();
return parsePlugin(document, pPath);
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/util/MimeUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/util/MimeUtil.java?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/util/MimeUtil.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/util/MimeUtil.java Tue Apr 30 19:36:23 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
// Tika imports
import org.apache.tika.Tika;
+import org.apache.tika.config.TikaConfig;
import org.apache.tika.mime.MimeType;
import org.apache.tika.mime.MimeTypeException;
import org.apache.tika.mime.MimeTypes;
@@ -33,7 +34,7 @@ import org.apache.tika.mime.MimeTypesFac
// Slf4j logging imports
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
+
// imported for Javadoc
import org.apache.nutch.protocol.ProtocolOutput;
@@ -169,11 +170,19 @@ public final class MimeUtil {
|| (type != null && type.getName().equals(MimeTypes.OCTET_STREAM))) {
// If no mime-type header, or cannot find a corresponding registered
// mime-type, then guess a mime-type from the url pattern
- type = this.mimeTypes.getMimeType(url) != null ? this.mimeTypes
- .getMimeType(url) : type;
- }
- retType= type.getName();
+ try {
+ TikaConfig tikaConfig = TikaConfig.getDefaultConfig();
+ Tika tika = new Tika(tikaConfig);
+ retType = tika.detect(url) != null ? tika.detect(url) : null;
+ } catch (Exception e) {
+ String message = "Problem loading default Tika configuration";
+ LOG.error(message, e);
+ throw new RuntimeException(e);
+ }
+ } else {
+ retType = type.getName();
+ }
// if magic is enabled use mime magic to guess if the mime type returned
// from the magic guess is different than the one that's already set so far
@@ -257,6 +266,4 @@ public final class MimeUtil {
return null;
}
}
-
-
}
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
(original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
Tue Apr 30 19:36:23 2013
@@ -50,6 +50,7 @@ import org.apache.nutch.util.MimeUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.TableUtil;
import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MimeType;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
@@ -164,7 +165,8 @@ public class TikaParser implements org.a
// populate Nutch metadata with Tika metadata
String[] TikaMDNames = tikamd.names();
for (String tikaMDName : TikaMDNames) {
- if (tikaMDName.equalsIgnoreCase(Metadata.TITLE)) continue;
+ if (tikaMDName.equalsIgnoreCase(TikaCoreProperties.TITLE.toString()))
+ continue;
// TODO what if multivalued?
page.putToMetadata(new Utf8(tikaMDName), ByteBuffer.wrap(Bytes.toBytes(tikamd
.get(tikaMDName))));
Modified: nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
(original)
+++ nutch/branches/2.x/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/DOMContentUtilsTest.java
Tue Apr 30 19:36:23 2013
@@ -29,7 +29,7 @@ import org.apache.nutch.util.NutchConfig
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.parser.Parser;
-import org.apache.xml.serialize.DOMSerializerImpl;
+import org.apache.xml.serializer.dom3.LSSerializerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -229,9 +229,9 @@ public class DOMContentUtilsTest extends
fail("caught exception: " + e);
}
testDOMs[i] = root;
- DOMSerializerImpl ds = new DOMSerializerImpl();
+ LSSerializerImpl lsi = new LSSerializerImpl();
System.out.println("input " + i + ": '" + testPages[i] + "'");
- System.out.println("output " + i + ": '" + ds.writeToString(root)
+ System.out.println("output " + i + ": '" + lsi.writeToString(root)
+ "'");
}
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
(original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
Tue Apr 30 19:36:23 2013
@@ -143,7 +143,7 @@ public class FileResponse {
if (!f.equals(f.getCanonicalFile())) {
// set headers
//hdrs.put("Location", f.getCanonicalFile().toURI());
- headers.set(Response.LOCATION, f.getCanonicalFile().toURL().toString());
+ headers.set(Response.LOCATION, f.getCanonicalFile().toURI().toURL().toString());
this.code = 300; // http redirect
return;
Modified: nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java?rev=1477792&r1=1477791&r2=1477792&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
(original)
+++ nutch/branches/2.x/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
Tue Apr 30 19:36:23 2013
@@ -45,6 +45,8 @@ import org.apache.commons.httpclient.NTC
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.httpclient.protocol.Protocol;
+import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
+import org.apache.commons.httpclient.protocol.SSLProtocolSocketFactory;
// Nutch imports
import org.apache.nutch.storage.WebPage;
@@ -124,7 +126,7 @@ public class Http extends HttpBase {
*/
public void setConf(Configuration conf) {
super.setConf(conf);
- this.conf = conf;
+ Http.conf = conf;
this.maxThreadsTotal = conf.getInt("fetcher.threads.fetch", 10);
this.proxyUsername = conf.get("http.proxy.username", "");
this.proxyPassword = conf.get("http.proxy.password", "");
@@ -178,8 +180,8 @@ public class Http extends HttpBase {
private void configureClient() {
// Set up an HTTPS socket factory that accepts self-signed certs.
- Protocol https = new Protocol("https",
- new DummySSLProtocolSocketFactory(), 443);
+ ProtocolSocketFactory factory = new SSLProtocolSocketFactory();
+ Protocol https = new Protocol("https", factory, 443);
Protocol.registerProtocol("https", https);
HttpConnectionManagerParams params = connectionManager.getParams();
@@ -195,7 +197,7 @@ public class Http extends HttpBase {
client.getParams().setConnectionManagerTimeout(timeout);
HostConfiguration hostConf = client.getHostConfiguration();
- ArrayList headers = new ArrayList();
+ ArrayList<Header> headers = new ArrayList<Header>();
// Set the User Agent in the header
headers.add(new Header("User-Agent", userAgent));
// prefer English
@@ -222,7 +224,7 @@ public class Http extends HttpBase {
this.proxyPort, this.proxyRealm);
NTCredentials proxyCredentials = new NTCredentials(
- this.proxyUsername, this.proxyPassword, this.agentHost,
+ this.proxyUsername, this.proxyPassword, Http.agentHost,
this.proxyRealm);
client.getState().setProxyCredentials(proxyAuthScope,
Author: tejasp
Date: Mon Apr 29 20:26:52 2013
New Revision: 1477319
URL: http://svn.apache.org/r1477319
Log:
NUTCH-1031 Delegate parsing of robots.txt to crawler-commons
Added:
nutch/branches/2.x/src/java/org/apache/nutch/protocol/RobotRulesParser.java
nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
Removed:
nutch/branches/2.x/src/java/org/apache/nutch/protocol/EmptyRobotRules.java
nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/ivy/ivy.xml
nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java
nutch/branches/2.x/src/java/org/apache/nutch/protocol/Protocol.java
nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
nutch/branches/2.x/src/plugin/protocol-sftp/src/java/org/apache/nutch/protocol/sftp/Sftp.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Mon Apr 29 20:26:52 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
+
* NUTCH-346 Improve readability of logs/hadoop.log (Renaud Richardet via tejasp)
* NUTCH-1501 Harmonize behavior of parsechecker and indexchecker (snagel + lewismc)
Modified: nutch/branches/2.x/ivy/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/ivy/ivy.xml?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/ivy/ivy.xml (original)
+++ nutch/branches/2.x/ivy/ivy.xml Mon Apr 29 20:26:52 2013
@@ -70,6 +70,7 @@
<dependency org="org.jdom" name="jdom" rev="1.1" conf="*->default" />
<dependency org="com.google.guava" name="guava" rev="11.0.2" />
+ <dependency org="com.google.code.crawler-commons" name="crawler-commons" rev="0.2"
/>
<!--Configuration: test -->
Modified: nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/fetcher/FetcherReducer.java Mon Apr 29 20:26:52
2013
@@ -61,6 +61,8 @@ import org.apache.nutch.util.TableUtil;
import org.apache.nutch.util.URLUtil;
import org.slf4j.Logger;
+import crawlercommons.robots.BaseRobotRules;
+
public class FetcherReducer
extends GoraReducer<IntWritable, FetchEntry, String, WebPage> {
@@ -152,9 +154,6 @@ extends GoraReducer<IntWritable, FetchEn
return "FetchItem [queueID=" + queueID + ", url=" + url + ", u=" + u
+ ", page=" + page + "]";
}
-
-
-
}
/**
@@ -489,8 +488,8 @@ extends GoraReducer<IntWritable, FetchEn
// fetch the page
final Protocol protocol = this.protocolFactory.getProtocol(fit.url);
- final RobotRules rules = protocol.getRobotRules(fit.url, fit.page);
- if (!rules.isAllowed(fit.u)) {
+ final BaseRobotRules rules = protocol.getRobotRules(fit.url, fit.page);
+ if (!rules.isAllowed(fit.u.toString())) {
// unblock
fetchQueues.finishFetchItem(fit, true);
if (LOG.isDebugEnabled()) {
Modified: nutch/branches/2.x/src/java/org/apache/nutch/protocol/Protocol.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/protocol/Protocol.java?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/protocol/Protocol.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/protocol/Protocol.java Mon Apr 29 20:26:52
2013
@@ -23,6 +23,8 @@ import org.apache.hadoop.conf.Configurab
import org.apache.nutch.plugin.FieldPluggable;
import org.apache.nutch.storage.WebPage;
+import crawlercommons.robots.BaseRobotRules;
+
/** A retriever of url content. Implemented by protocol extensions. */
public interface Protocol extends FieldPluggable, Configurable {
/** The name of the extension point. */
@@ -46,7 +48,8 @@ public interface Protocol extends FieldP
*/
public final static String CHECK_ROBOTS = "protocol.plugin.check.robots";
- /** Returns the {@link Content} for a fetchlist entry.
+ /*
+ * Returns the {@link Content} for a fetchlist entry.
*/
ProtocolOutput getProtocolOutput(String url, WebPage page);
@@ -56,5 +59,5 @@ public interface Protocol extends FieldP
* @param page
* @return robot rules (specific for this url or default), never null
*/
- RobotRules getRobotRules(String url, WebPage page);
+ BaseRobotRules getRobotRules(String url, WebPage page);
}
Added: nutch/branches/2.x/src/java/org/apache/nutch/protocol/RobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/protocol/RobotRulesParser.java?rev=1477319&view=auto
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/protocol/RobotRulesParser.java (added)
+++ nutch/branches/2.x/src/java/org/apache/nutch/protocol/RobotRulesParser.java Mon Apr 29
20:26:52 2013
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.protocol;
+
+// JDK imports
+import java.io.File;
+import java.io.FileReader;
+import java.io.LineNumberReader;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Hashtable;
+import java.util.StringTokenizer;
+
+// Commons Logging imports
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// Nutch imports
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.io.Files;
+
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
+import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
+import crawlercommons.robots.SimpleRobotRulesParser;
+
+/**
+ * This class uses crawler-commons for handling the parsing of {@code robots.txt} files.
+ * It emits SimpleRobotRules objects, which describe the download permissions
+ * as described in SimpleRobotRulesParser.
+ */
+public abstract class RobotRulesParser implements Configurable {
+
+ public static final Logger LOG = LoggerFactory.getLogger(RobotRulesParser.class);
+
+ protected static final Hashtable<String, BaseRobotRules> CACHE = new Hashtable<String,
BaseRobotRules> ();
+
+ /**
+ * A {@link BaseRobotRules} object appropriate for use
+ * when the {@code robots.txt} file is empty or missing;
+ * all requests are allowed.
+ */
+ public static final BaseRobotRules EMPTY_RULES = new SimpleRobotRules(RobotRulesMode.ALLOW_ALL);
+
+ /**
+ * A {@link BaseRobotRules} object appropriate for use when the
+ * {@code robots.txt} file is not fetched due to a {@code 403/Forbidden}
+ * response; all requests are disallowed.
+ */
+ public static BaseRobotRules FORBID_ALL_RULES = new SimpleRobotRules(RobotRulesMode.ALLOW_NONE);
+
+ private static SimpleRobotRulesParser robotParser = new SimpleRobotRulesParser();
+ private Configuration conf;
+ protected String agentNames;
+
+ public RobotRulesParser() { }
+
+ public RobotRulesParser(Configuration conf) {
+ setConf(conf);
+ }
+
+ /**
+ * Set the {@link Configuration} object
+ */
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+
+ // Grab the agent names we advertise to robots files.
+ String agentName = conf.get("http.agent.name");
+ if (null == agentName) {
+ throw new RuntimeException("Agent name not configured!");
+ }
+
+ String agentNames = conf.get("http.robots.agents");
+ StringTokenizer tok = new StringTokenizer(agentNames, ",");
+ ArrayList<String> agents = new ArrayList<String>();
+ while (tok.hasMoreTokens()) {
+ agents.add(tok.nextToken().trim());
+ }
+
+ /**
+ * If there are no agents for robots-parsing, use the
+ * default agent-string. If both are present, our agent-string
+ * should be the first one we advertise to robots-parsing.
+ */
+ if (agents.size() == 0) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("No agents listed in 'http.robots.agents' property!");
+ }
+ } else {
+ StringBuffer combinedAgentsString = new StringBuffer(agentName);
+ int index = 0;
+
+ if ((agents.get(0)).equalsIgnoreCase(agentName))
+ index++;
+ else if (LOG.isErrorEnabled()) {
+ LOG.error("Agent we advertise (" + agentName
+ + ") not listed first in 'http.robots.agents' property!");
+ }
+
+ // append all the agents from the http.robots.agents property
+ for(; index < agents.size(); index++) {
+ combinedAgentsString.append(", " + agents.get(index));
+ }
+
+ // always make sure "*" is included in the end
+ combinedAgentsString.append(", *");
+ this.agentNames = combinedAgentsString.toString();
+ }
+ }
+
+ /**
+ * Get the {@link Configuration} object
+ */
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Parses the robots content using the {@link SimpleRobotRulesParser} from crawler commons
+ *
+ * @param url A string containing url
+ * @param content Contents of the robots file in a byte array
+ * @param contentType The
+ * @param robotName A string containing value of
+ * @return BaseRobotRules object
+ */
+ public BaseRobotRules parseRules (String url, byte[] content, String contentType, String
robotName) {
+ return robotParser.parseContent(url, content, contentType, robotName);
+ }
+
+ public BaseRobotRules getRobotRulesSet(Protocol protocol, String url) {
+ URL u = null;
+ try {
+ u = new URL(url);
+ } catch (Exception e) {
+ return EMPTY_RULES;
+ }
+ return getRobotRulesSet(protocol, u);
+ }
+
+ public abstract BaseRobotRules getRobotRulesSet(Protocol protocol, URL url);
+
+ /** command-line main for testing */
+ public static void main(String[] argv) {
+
+ if (argv.length < 3) {
+ System.err.println("Usage: RobotRulesParser <robots-file> <url-file> <agent-names>\n");
+ System.err.println(" <robots-file> - Input robots.txt file which will be parsed.");
+ System.err.println(" <url-file> - Contains input URLs (1 per line) which
are tested against the rules.");
+ System.err.println(" <agent-names> - Input agent name. Multiple agent names
can be specified using spaces.");
+ System.exit(-1);
+ }
+
+ try {
+ StringBuilder agentNames = new StringBuilder();
+ for(int counter = 2; counter < argv.length; counter++)
+ agentNames.append(argv[counter]).append(",");
+
+ agentNames.deleteCharAt(agentNames.length()-1);
+
+ byte[] robotsBytes = Files.toByteArray(new File(argv[0]));
+ BaseRobotRules rules = robotParser.parseContent(argv[0], robotsBytes, "text/plain",
agentNames.toString());
+
+ LineNumberReader testsIn = new LineNumberReader(new FileReader(argv[1]));
+ String testPath = testsIn.readLine().trim();
+ while (testPath != null) {
+ System.out.println( (rules.isAllowed(testPath) ? "allowed" : "not allowed") +
+ ":\t" + testPath);
+ testPath = testsIn.readLine();
+ }
+ testsIn.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
Modified: nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
(original)
+++ nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
Mon Apr 29 20:26:52 2013
@@ -32,23 +32,21 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
import org.apache.nutch.protocol.ProtocolStatusUtils;
-import org.apache.nutch.protocol.RobotRules;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.GZIPUtils;
import org.apache.nutch.util.DeflateUtils;
import org.apache.nutch.util.MimeUtil;
-/**
- * @author Jérôme Charron
- */
-public abstract class HttpBase implements Protocol {
+// crawler-commons imports
+import crawlercommons.robots.BaseRobotRules;
+public abstract class HttpBase implements Protocol {
public static final int BUFFER_SIZE = 8 * 1024;
private static final byte[] EMPTY_CONTENT = new byte[0];
- private RobotRulesParser robots = null;
+ private HttpRobotRulesParser robots = null;
/** The proxy hostname. */
protected String proxyHost = null;
@@ -102,7 +100,7 @@ public abstract class HttpBase implement
if (logger != null) {
this.logger = logger;
}
- robots = new RobotRulesParser();
+ robots = new HttpRobotRulesParser();
}
// Inherited Javadoc
@@ -128,13 +126,10 @@ public abstract class HttpBase implement
return this.conf;
}
-
-
public ProtocolOutput getProtocolOutput(String url, WebPage page) {
try {
URL u = new URL(url);
- String host = null;
Response response = getResponse(u, page, false); // make a request
int code = response.getCode();
byte[] content = response.getContent();
@@ -145,7 +140,6 @@ public abstract class HttpBase implement
if (code == 200) { // got a good response
return new ProtocolOutput(c); // return it
-
} else if (code == 410) { // page is gone
return new ProtocolOutput(c,
ProtocolStatusUtils.makeStatus(ProtocolStatusCodes.GONE, "Http: " + code + "
url=" + url));
@@ -206,8 +200,6 @@ public abstract class HttpBase implement
/* -------------------------- *
* </implementation:Protocol> *
* -------------------------- */
-
-
public String getProxyHost() {
return proxyHost;
}
@@ -367,10 +359,6 @@ public abstract class HttpBase implement
url = args[i];
}
- // if (verbose) {
- // LOGGER.setLevel(Level.FINE);
- // }
-
ProtocolOutput out = http.getProtocolOutput(url, new WebPage());
Content content = out.getContent();
@@ -383,17 +371,14 @@ public abstract class HttpBase implement
String text = new String(content.getContent());
System.out.println(text);
}
-
}
-
protected abstract Response getResponse(URL url,
WebPage page, boolean followRedirects)
throws ProtocolException, IOException;
@Override
- public RobotRules getRobotRules(String url, WebPage page) {
+ public BaseRobotRules getRobotRules(String url, WebPage page) {
return robots.getRobotRulesSet(this, url);
}
-
}
Added: nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java?rev=1477319&view=auto
==============================================================================
--- nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
(added)
+++ nutch/branches/2.x/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
Mon Apr 29 20:26:52 2013
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.protocol.http.api;
+
+import java.net.URL;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.protocol.Protocol;
+import org.apache.nutch.protocol.RobotRulesParser;
+import org.apache.nutch.storage.WebPage;
+
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
+
+/**
+ * This class is used for parsing robots for urls belonging to HTTP protocol.
+ * It extends the generic {@link RobotRulesParser} class and contains
+ * Http protocol specific implementation for obtaining the robots file.
+ */
+public class HttpRobotRulesParser extends RobotRulesParser {
+
+ public static final Logger LOG = LoggerFactory.getLogger(HttpRobotRulesParser.class);
+ protected boolean allowForbidden = false;
+
+ HttpRobotRulesParser() { }
+
+ public HttpRobotRulesParser(Configuration conf) {
+ super(conf);
+ allowForbidden = conf.getBoolean("http.robots.403.allow", false);
+ }
+
+ /**
+ * The hosts for which the caching of robots rules is yet to be done,
+ * it sends a Http request to the host corresponding to the {@link URL}
+ * passed, gets robots file, parses the rules and caches the rules object
+ * to avoid re-work in future.
+ *
+ * @param http The {@link Protocol} object
+ * @param url URL
+ *
+ * @return robotRules A {@link BaseRobotRules} object for the rules
+ */
+ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
+
+ String protocol = url.getProtocol().toLowerCase(); // normalize to lower case
+ String host = url.getHost().toLowerCase(); // normalize to lower case
+
+ BaseRobotRules robotRules = (SimpleRobotRules)CACHE.get(protocol + ":" + host);
+
+ boolean cacheRule = true;
+
+ if (robotRules == null) { // cache miss
+ URL redir = null;
+ if (LOG.isTraceEnabled()) { LOG.trace("cache miss " + url); }
+ try {
+ Response response = ((HttpBase)http).getResponse(new URL(url, "/robots.txt"),
+ new WebPage(), true);
+ // try one level of redirection ?
+ if (response.getCode() == 301 || response.getCode() == 302) {
+ String redirection = response.getHeader("Location");
+ if (redirection == null) {
+ // some versions of MS IIS are known to mangle this header
+ redirection = response.getHeader("location");
+ }
+ if (redirection != null) {
+ if (!redirection.startsWith("http")) {
+ // RFC says it should be absolute, but apparently it isn't
+ redir = new URL(url, redirection);
+ } else {
+ redir = new URL(redirection);
+ }
+
+ response = ((HttpBase)http).getResponse(redir, new WebPage(), true);
+ }
+ }
+
+ if (response.getCode() == 200) // found rules: parse them
+ robotRules = parseRules(url.toString(), response.getContent(),
+ response.getHeader("Content-Type"),
+ agentNames);
+
+ else if ( (response.getCode() == 403) && (!allowForbidden) )
+ robotRules = FORBID_ALL_RULES; // use forbid all
+ else if (response.getCode() >= 500) {
+ cacheRule = false;
+ robotRules = EMPTY_RULES;
+ }else
+ robotRules = EMPTY_RULES; // use default rules
+ } catch (Throwable t) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Couldn't get robots.txt for " + url + ": " + t.toString());
+ }
+ cacheRule = false;
+ robotRules = EMPTY_RULES;
+ }
+
+ if (cacheRule) {
+ CACHE.put(protocol + ":" + host, robotRules); // cache rules for host
+ if (redir != null && !redir.getHost().equals(host)) {
+ // cache also for the redirected host
+ CACHE.put(protocol + ":" + redir.getHost(), robotRules);
+ }
+ }
+ }
+ return robotRules;
+ }
+}
Modified: nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
(original)
+++ nutch/branches/2.x/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
Mon Apr 29 20:26:52 2013
@@ -17,292 +17,100 @@
package org.apache.nutch.protocol.http.api;
-import org.apache.nutch.protocol.http.api.RobotRulesParser.RobotRuleSet;
-
+import crawlercommons.robots.BaseRobotRules;
import junit.framework.TestCase;
+/**
+ * JUnit test case which tests
+ * 1. that robots filtering is performed correctly as per the agent name
+ * 2. that crawl delay is extracted correctly from the robots file
+ *
+ */
public class TestRobotRulesParser extends TestCase {
- private static final String LF= "\n";
- private static final String CR= "\r";
- private static final String CRLF= "\r\n";
+
+ private static final String CONTENT_TYPE = "text/plain";
+ private static final String SINGLE_AGENT = "Agent1";
+ private static final String MULTIPLE_AGENTS = "Agent2, Agent1";
+ private static final String UNKNOWN_AGENT = "AgentABC";
+ private static final String CR = "\r";
- private static final boolean[] ACCEPT_ALL = {
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- true, // "/d",
- true, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- true, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- };
+ private static final String ROBOTS_STRING =
+ "User-Agent: Agent1 #foo" + CR
+ + "Disallow: /a" + CR
+ + "Disallow: /b/a" + CR
+ + "#Disallow: /c" + CR
+ + "Crawl-delay: 10" + CR // set crawl delay for Agent1 as 10 sec
+ + "" + CR
+ + "" + CR
+ + "User-Agent: Agent2" + CR
+ + "Disallow: /a/bloh" + CR
+ + "Disallow: /c" + CR
+ + "Disallow: /foo" + CR
+ + "Crawl-delay: 20" + CR
+ + "" + CR
+ + "User-Agent: *" + CR
+ + "Disallow: /foo/bar/" + CR; // no crawl delay for other agents
- private static final String[] ROBOTS_STRINGS= new String[] {
- "User-Agent: Agent1 #foo" + CR
- + "Disallow: /a" + CR
- + "Disallow: /b/a" + CR
- + "#Disallow: /c" + CR
- + "" + CR
- + "" + CR
- + "User-Agent: Agent2 Agent3#foo" + CR
- + "User-Agent: Agent4" + CR
- + "Disallow: /d" + CR
- + "Disallow: /e/d/" + CR
- + "" + CR
- + "User-Agent: *" + CR
- + "Disallow: /foo/bar/" + CR,
- null // Used to test EMPTY_RULES
+ private static final String[] TEST_PATHS = new String[] {
+ "http://example.com/a",
+ "http://example.com/a/bloh/foo.html",
+ "http://example.com/b",
+ "http://example.com/c",
+ "http://example.com/b/a/index.html",
+ "http://example.com/foo/bar/baz.html"
+ };
+
+ private static final boolean[] RESULTS = new boolean[] {
+ false, // /a
+ false, // /a/bloh/foo.html
+ true, // /b
+ true, // /c
+ false, // /b/a/index.html
+ true // /foo/bar/baz.html
};
- private static final String[] AGENT_STRINGS= new String[] {
- "Agent1",
- "Agent2",
- "Agent3",
- "Agent4",
- "Agent5",
- };
-
- private static final boolean[][] NOT_IN_ROBOTS_STRING= new boolean[][] {
- {
- false,
- false,
- false,
- false,
- true,
- },
- {
- false,
- false,
- false,
- false,
- true,
- }
- };
+ private HttpRobotRulesParser parser;
+ private BaseRobotRules rules;
- private static final String[] TEST_PATHS= new String[] {
- "/a",
- "/a/",
- "/a/bloh/foo.html",
- "/b",
- "/b/a",
- "/b/a/index.html",
- "/b/b/foo.html",
- "/c",
- "/c/a",
- "/c/a/index.html",
- "/c/b/foo.html",
- "/d",
- "/d/a",
- "/e/a/index.html",
- "/e/d",
- "/e/d/foo.html",
- "/e/doh.html",
- "/f/index.html",
- "/foo/bar/baz.html",
- "/f/",
- };
-
- private static final boolean[][][] ALLOWED= new boolean[][][] {
- { // ROBOTS_STRINGS[0]
- { // Agent1
- false, // "/a",
- false, // "/a/",
- false, // "/a/bloh/foo.html"
- true, // "/b",
- false, // "/b/a",
- false, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- true, // "/d",
- true, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- true, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent2
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- false, // "/d",
- false, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- false, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent3
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- false, // "/d",
- false, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- false, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent4
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- false, // "/d",
- false, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- false, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent5/"*"
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- true, // "/d",
- true, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- true, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- false, // "/foo/bar.html",
- true, // "/f/",
- }
- },
- { // ROBOTS_STRINGS[1]
- ACCEPT_ALL, // Agent 1
- ACCEPT_ALL, // Agent 2
- ACCEPT_ALL, // Agent 3
- ACCEPT_ALL, // Agent 4
- ACCEPT_ALL, // Agent 5
- }
- };
-
public TestRobotRulesParser(String name) {
super(name);
+ parser = new HttpRobotRulesParser();
}
- public void testRobotsOneAgent() {
- for (int i= 0; i < ROBOTS_STRINGS.length; i++) {
- for (int j= 0; j < AGENT_STRINGS.length; j++) {
- testRobots(i, new String[] { AGENT_STRINGS[j] },
- TEST_PATHS, ALLOWED[i][j]);
- }
+ /**
+ * Test that the robots rules are interpreted correctly by the robots rules parser.
+ */
+ public void testRobotsAgent() {
+ rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(), CONTENT_TYPE,
SINGLE_AGENT);
+
+ for(int counter = 0; counter < TEST_PATHS.length; counter++) {
+ assertTrue("testing on agent (" + SINGLE_AGENT + "), and "
+ + "path " + TEST_PATHS[counter]
+ + " got " + rules.isAllowed(TEST_PATHS[counter]),
+ rules.isAllowed(TEST_PATHS[counter]) == RESULTS[counter]);
}
- }
- public void testRobotsTwoAgents() {
- for (int i= 0; i < ROBOTS_STRINGS.length; i++) {
- for (int j= 0; j < AGENT_STRINGS.length; j++) {
- for (int k= 0; k < AGENT_STRINGS.length; k++) {
- int key= j;
- if (NOT_IN_ROBOTS_STRING[i][j])
- key= k;
- testRobots(i, new String[] { AGENT_STRINGS[j], AGENT_STRINGS[k] },
- TEST_PATHS, ALLOWED[i][key]);
- }
- }
- }
- }
-
- public void testCrawlDelay() {
- RobotRulesParser p = new RobotRulesParser(new String[] { "nutchbot" });
- String delayRule1 = "User-agent: nutchbot" + CR +
- "Crawl-delay: 10" + CR +
- "User-agent: foobot" + CR +
- "Crawl-delay: 20" + CR +
- "User-agent: *" + CR +
- "Disallow:/baz" + CR;
- String delayRule2 = "User-agent: foobot" + CR +
- "Crawl-delay: 20" + CR +
- "User-agent: *" + CR +
- "Disallow:/baz" + CR;
- RobotRuleSet rules = p.parseRules(delayRule1.getBytes());
- long crawlDelay = rules.getCrawlDelay();
- assertTrue("testing crawl delay for agent nutchbot - rule 1", (crawlDelay == 10000));
- rules = p.parseRules(delayRule2.getBytes());
- crawlDelay = rules.getCrawlDelay();
- assertTrue("testing crawl delay for agent nutchbot - rule 2", (crawlDelay == -1));
- }
+ rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(), CONTENT_TYPE,
MULTIPLE_AGENTS);
- // helper
-
- public void testRobots(int robotsString, String[] agents, String[] paths,
- boolean[] allowed) {
- String agentsString= agents[0];
- for (int i= 1; i < agents.length; i++)
- agentsString= agentsString + "," + agents[i];
- RobotRulesParser p= new RobotRulesParser(agents);
- RobotRuleSet rules= p.parseRules(ROBOTS_STRINGS[robotsString] != null
- ? ROBOTS_STRINGS[robotsString].getBytes()
- : null);
- for (int i= 0; i < paths.length; i++) {
- assertTrue("testing robots file "+robotsString+", on agents ("
- + agentsString + "), and path " + TEST_PATHS[i] + "; got "
- + rules.isAllowed(TEST_PATHS[i]) + ", rules are: " + LF
- + rules,
- rules.isAllowed(TEST_PATHS[i]) == allowed[i]);
+ for(int counter = 0; counter < TEST_PATHS.length; counter++) {
+ assertTrue("testing on agents (" + MULTIPLE_AGENTS + "), and "
+ + "path " + TEST_PATHS[counter]
+ + " got " + rules.isAllowed(TEST_PATHS[counter]),
+ rules.isAllowed(TEST_PATHS[counter]) == RESULTS[counter]);
}
}
-
-
+ /**
+ * Test that the crawl delay is extracted from the robots file for respective agent.
+ * If its not specified for a given agent, default value must be returned.
+ */
+ public void testCrawlDelay() {
+ // for SINGLE_AGENT, the crawl delay of 10 sec ie. 10000 msec must be returned by the
parser
+ rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(), CONTENT_TYPE, SINGLE_AGENT);
+ assertTrue("testing crawl delay for agent "+ SINGLE_AGENT +" : ", (rules.getCrawlDelay()
== 10000));
+
+ // for UNKNOWN_AGENT, the default crawl delay must be returned.
+ rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(), CONTENT_TYPE, UNKNOWN_AGENT);
+ assertTrue("testing crawl delay for agent "+ UNKNOWN_AGENT +" : ", (rules.getCrawlDelay()
== Long.MIN_VALUE));
+ }
}
Modified: nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
(original)
+++ nutch/branches/2.x/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
Mon Apr 29 20:26:52 2013
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.nutch.protocol.file;
import java.net.URL;
@@ -23,28 +22,30 @@ import java.util.HashSet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.EmptyRobotRules;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
import org.apache.nutch.protocol.ProtocolStatusUtils;
-import org.apache.nutch.protocol.RobotRules;
+import org.apache.nutch.protocol.RobotRulesParser;
import org.apache.nutch.storage.ProtocolStatus;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.NutchConfiguration;
-/************************************
- * File.java deals with file: scheme.
- *
- * Configurable parameters are defined under "FILE properties" section in
- * ./conf/nutch-default.xml or similar.
- *
- * @author John Xing
- ***********************************/
+import crawlercommons.robots.BaseRobotRules;
+
+/**
+ * This class is a protocol plugin used for file: scheme.
+ * It creates {@link FileResponse} object and gets the content of the url from it.
+ * Configurable parameters are {@code file.content.limit} and {@code file.crawl.parent}
+ * in nutch-default.xml defined under "file properties" section.
+ */
public class File implements Protocol {
public static final Logger LOG = LoggerFactory.getLogger(File.class);
@@ -65,14 +66,40 @@ public class File implements Protocol {
private Configuration conf;
// constructor
- public File() {
- }
+ public File() { }
- /** Set the point at which content is truncated. */
- public void setMaxContentLength(int length) {
- maxContentLength = length;
+ /**
+ * Set the {@link Configuration} object
+ */
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ this.maxContentLength = conf.getInt("file.content.limit", 64 * 1024);
+ this.crawlParents = conf.getBoolean("file.crawl.parent", true);
}
-
+
+ /**
+ * Get the {@link Configuration} object
+ */
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * Set the point at which content is truncated.
+ */
+ public void setMaxContentLength(int maxContentLength) {
+ this.maxContentLength = maxContentLength;
+ }
+
+ /**
+ * Creates a {@link FileResponse} object corresponding to the url and
+ * return a {@link ProtocolOutput} object as per the content received
+ *
+ * @param url Text containing the url
+ * @param datum The CrawlDatum object corresponding to the url
+ *
+ * @return {@link ProtocolOutput} object for the content of the file indicated by url
+ */
public ProtocolOutput getProtocolOutput(String url, WebPage page) {
String urlString = url.toString();
try {
@@ -82,16 +109,11 @@ public class File implements Protocol {
while (true) {
FileResponse response;
- response = new FileResponse(u, page, this, getConf()); // make
- // a
- // request
-
+ response = new FileResponse(u, page, this, getConf()); // make a request
int code = response.getCode();
if (code == 200) { // got a good response
- return new ProtocolOutput(response.toContent()); // return
- // it
-
+ return new ProtocolOutput(response.toContent()); // return it
} else if (code >= 300 && code < 400) { // handle redirect
if (redirects == MAX_REDIRECTS)
throw new FileException("Too many redirects: " + url);
@@ -114,16 +136,13 @@ public class File implements Protocol {
}
@Override
- public RobotRules getRobotRules(String url, WebPage page) {
- return EmptyRobotRules.RULES;
- }
-
- @Override
public Collection<Field> getFields() {
return FIELDS;
}
- /** For debugging. */
+ /**
+ * Quick way for running this class. Useful for debugging.
+ */
public static void main(String[] args) throws Exception {
int maxContentLength = Integer.MIN_VALUE;
boolean dumpContent = false;
@@ -154,9 +173,6 @@ public class File implements Protocol {
if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength
file.setMaxContentLength(maxContentLength);
- // set log level
- // LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase()));
-
Content content = file.getProtocolOutput(urlString, new WebPage())
.getContent();
@@ -172,13 +188,11 @@ public class File implements Protocol {
file = null;
}
- public void setConf(Configuration conf) {
- this.conf = conf;
- this.maxContentLength = conf.getInt("file.content.limit", 64 * 1024);
- this.crawlParents = conf.getBoolean("file.crawl.parent", true);
- }
-
- public Configuration getConf() {
- return this.conf;
- }
+ /**
+ * No robots parsing is done for file protocol.
+ * So this returns a set of empty rules which will allow every url.
+ */
+ public BaseRobotRules getRobotRules(String url, WebPage page) {
+ return RobotRulesParser.EMPTY_RULES;
+ }
}
Modified: nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
(original)
+++ nutch/branches/2.x/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
Mon Apr 29 20:26:52 2013
@@ -28,23 +28,27 @@ import org.apache.commons.net.ftp.FTPFil
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.EmptyRobotRules;
+import org.apache.nutch.protocol.RobotRulesParser;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatusCodes;
import org.apache.nutch.protocol.ProtocolStatusUtils;
-import org.apache.nutch.protocol.RobotRules;
import org.apache.nutch.storage.ProtocolStatus;
import org.apache.nutch.storage.WebPage;
-/************************************
- * Ftp.java deals with ftp: scheme.
- *
- * Configurable parameters are defined under "FTP properties" section in
- * ./conf/nutch-default.xml or similar.
+import crawlercommons.robots.BaseRobotRules;
+
+/**
+ * This class is a protocol plugin used for ftp: scheme.
+ * It creates {@link FtpResponse} object and gets the content of the url from it.
+ * Configurable parameters are {@code ftp.username}, {@code ftp.password},
+ * {@code ftp.content.limit}, {@code ftp.timeout},
+ * {@code ftp.server.timeout}, {@code ftp.password},
+ * {@code ftp.keep.connection} and {@code ftp.follow.talk}.
+ * For details see "FTP properties" section in {@code nutch-default.xml}.
*
* @author John Xing
- ***********************************/
+ */
public class Ftp implements Protocol {
public static final Logger LOG = LoggerFactory.getLogger(Ftp.class);
@@ -109,6 +113,15 @@ public class Ftp implements Protocol {
this.keepConnection = keepConnection;
}
+ /**
+ * Creates a {@link FtpResponse} object corresponding to the url and
+ * returns a {@link ProtocolOutput} object as per the content received
+ *
+ * @param url Text containing the ftp url
+ * @param datum The CrawlDatum object corresponding to the url
+ *
+ * @return {@link ProtocolOutput} object for the url
+ */
public ProtocolOutput getProtocolOutput(String url, WebPage page) {
try {
URL u = new URL(url);
@@ -154,6 +167,9 @@ public class Ftp implements Protocol {
}
}
+ /**
+ * Set the {@link Configuration} object
+ */
public void setConf(Configuration conf) {
this.conf = conf;
this.maxContentLength = conf.getInt("ftp.content.limit", 64 * 1024);
@@ -165,14 +181,13 @@ public class Ftp implements Protocol {
this.followTalk = conf.getBoolean("ftp.follow.talk", false);
}
+ /**
+ * Get the {@link Configuration} object
+ */
public Configuration getConf() {
return this.conf;
}
- public RobotRules getRobotRules(String url, WebPage page) {
- return EmptyRobotRules.RULES;
- }
-
/** For debugging. */
public static void main(String[] args) throws Exception {
int timeout = Integer.MIN_VALUE;
@@ -222,9 +237,6 @@ public class Ftp implements Protocol {
if (maxContentLength != Integer.MIN_VALUE) // set maxContentLength
ftp.setMaxContentLength(maxContentLength);
- // set log level
- // LOG.setLevel(Level.parse((new String(logLevel)).toUpperCase()));
-
Content content = ftp.getProtocolOutput(urlString, new WebPage())
.getContent();
@@ -244,4 +256,12 @@ public class Ftp implements Protocol {
return FIELDS;
}
+ /**
+ * Currently, no robots parsing is done for ftp protocol
+ * and this returns a set of empty rules which will allow every url.
+ * There a jira logged for the same NUTCH-1513
+ */
+ public BaseRobotRules getRobotRules(String url, WebPage page) {
+ return RobotRulesParser.EMPTY_RULES;
+ }
}
Modified: nutch/branches/2.x/src/plugin/protocol-sftp/src/java/org/apache/nutch/protocol/sftp/Sftp.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/protocol-sftp/src/java/org/apache/nutch/protocol/sftp/Sftp.java?rev=1477319&r1=1477318&r2=1477319&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/protocol-sftp/src/java/org/apache/nutch/protocol/sftp/Sftp.java
(original)
+++ nutch/branches/2.x/src/plugin/protocol-sftp/src/java/org/apache/nutch/protocol/sftp/Sftp.java
Mon Apr 29 20:26:52 2013
@@ -38,9 +38,9 @@ import org.apache.nutch.net.protocols.Re
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
-import org.apache.nutch.protocol.RobotRules;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
+import org.apache.nutch.protocol.RobotRulesParser;
//JSCH imports
import com.jcraft.jsch.ChannelSftp;
@@ -50,6 +50,8 @@ import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
import com.jcraft.jsch.ChannelSftp.LsEntry;
+import crawlercommons.robots.BaseRobotRules;
+
/**
* This class uses the Jsch package to fetch content using the Sftp protocol.
*
@@ -224,10 +226,16 @@ public class Sftp implements Protocol {
}
}
+ /**
+ * Get the {@link Configuration} object
+ */
public Configuration getConf() {
return configuration;
}
+ /**
+ * Set the {@link Configuration} object
+ */
public void setConf(Configuration arg0) {
configuration = arg0;
@@ -288,34 +296,9 @@ public class Sftp implements Protocol {
}
}
- /*
- * (non-Javadoc)
- *
- * @see org.apache.nutch.protocol.Protocol#getRobotRules(java.lang.String,
- * org.apache.nutch.storage.WebPage)
- */
@Override
- public RobotRules getRobotRules(String url, WebPage page) {
- return new RobotRules() {
-
- @Override
- public boolean isAllowed(URL url) {
- // they're all allowed for now.
- return true;
- }
-
- @Override
- public long getExpireTime() {
- // set to 0 for never expire
- return 0;
- }
-
- @Override
- public long getCrawlDelay() {
- // no delay
- return 0;
- }
- };
+ public BaseRobotRules getRobotRules(String url, WebPage page) {
+ return RobotRulesParser.EMPTY_RULES;
}
/*
@@ -327,5 +310,4 @@ public class Sftp implements Protocol {
public Collection<Field> getFields() {
return Collections.emptySet();
}
-
}
Author: tejasp
Date: Sun Apr 28 23:35:58 2013
New Revision: 1476861
URL: http://svn.apache.org/r1476861
Log:
NUTCH-346 Improve readability of logs/hadoop.log
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/conf/log4j.properties
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1476861&r1=1476860&r2=1476861&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Sun Apr 28 23:35:58 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-346 Improve readability of logs/hadoop.log (Renaud Richardet via tejasp)
+
* NUTCH-1501 Harmonize behavior of parsechecker and indexchecker (snagel + lewismc)
* NUTCH-1551 Improve WebTableReader field order and display batchId (lewismc)
Modified: nutch/branches/2.x/conf/log4j.properties
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/conf/log4j.properties?rev=1476861&r1=1476860&r2=1476861&view=diff
==============================================================================
--- nutch/branches/2.x/conf/log4j.properties (original)
+++ nutch/branches/2.x/conf/log4j.properties Sun Apr 28 23:35:58 2013
@@ -40,6 +40,7 @@ log4j.logger.org.apache.nutch.crawl.WebT
log4j.logger.org.apache.nutch.host.HostDbReader=INFO,cmdstdout
log4j.logger.org.apache.nutch.parse.ParserChecker=INFO,cmdstdout
log4j.logger.org.apache.nutch.indexer.IndexingFiltersChecker=INFO,cmdstdout
+log4j.logger.org.apache.nutch.plugin.PluginRepository=WARN
log4j.logger.org.apache.nutch=INFO
log4j.logger.org.apache.hadoop=WARN
Author: tejasp
Date: Sun Apr 28 23:32:59 2013
New Revision: 1476859
URL: http://svn.apache.org/r1476859
Log:
NUTCH-346 Improve readability of logs/hadoop.log
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/log4j.properties
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1476859&r1=1476858&r2=1476859&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sun Apr 28 23:32:59 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-346 Improve readability of logs/hadoop.log (Renaud Richardet via tejasp)
+
* NUTCH-829 duplicate hadoop temp files (Mike Baranczak, lewismc, tejasp)
* NUTCH-1501 Harmonize behavior of parsechecker and indexchecker (snagel + lewismc)
Modified: nutch/trunk/conf/log4j.properties
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/log4j.properties?rev=1476859&r1=1476858&r2=1476859&view=diff
==============================================================================
--- nutch/trunk/conf/log4j.properties (original)
+++ nutch/trunk/conf/log4j.properties Sun Apr 28 23:32:59 2013
@@ -35,6 +35,7 @@ log4j.logger.org.apache.nutch.indexer.In
log4j.logger.org.apache.nutch.tools.FreeGenerator=INFO,cmdstdout
log4j.logger.org.apache.nutch.util.domain.DomainStatistics=INFO,cmdstdout
log4j.logger.org.apache.nutch.tools.CrawlDBScanner=INFO,cmdstdout
+log4j.logger.org.apache.nutch.plugin.PluginRepository=WARN
log4j.logger.org.apache.nutch=INFO
log4j.logger.org.apache.hadoop=WARN
Author: tejasp
Date: Sun Apr 28 01:13:21 2013
New Revision: 1476702
URL: http://svn.apache.org/r1476702
Log:
NUTCH-829 duplicate hadoop temp files
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1476702&r1=1476701&r2=1476702&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Sun Apr 28 01:13:21 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-829 duplicate hadoop temp files (Mike Baranczak, lewismc, tejasp)
+
* NUTCH-1501 Harmonize behavior of parsechecker and indexchecker (snagel + lewismc)
* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
Modified: nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java?rev=1476702&r1=1476701&r2=1476702&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java Sun Apr 28 01:13:21 2013
@@ -492,7 +492,7 @@ public class Generator extends Configure
throws IOException {
Path tempDir = new Path(getConf().get("mapred.temp.dir", ".") + "/generate-temp-"
- + System.currentTimeMillis());
+ + java.util.UUID.randomUUID().toString());
Path lock = new Path(dbDir, CrawlDb.LOCK_NAME);
FileSystem fs = FileSystem.get(getConf());
@@ -582,7 +582,7 @@ public class Generator extends Configure
if (getConf().getBoolean(GENERATE_UPDATE_CRAWLDB, false)) {
// update the db from tempDir
Path tempDir2 = new Path(getConf().get("mapred.temp.dir", ".") + "/generate-temp-"
- + System.currentTimeMillis());
+ + java.util.UUID.randomUUID().toString());
job = new NutchJob(getConf());
job.setJobName("generate: updatedb " + dbDir);
Author: tejasp
Date: Thu Apr 25 05:05:56 2013
New Revision: 1475631
URL: http://svn.apache.org/r1475631
Log:
NUTCH-1565 Proper downloads page for Nutch (lewismc and tejasp)
Added:
nutch/site/forrest/src/documentation/content/xdocs/downloads.xml
nutch/site/publish/downloads.html
nutch/site/publish/downloads.pdf (with props)
nutch/site/publish/skin/images/apache-thanks.png (with props)
nutch/site/publish/skin/images/built-with-cocoon.gif (with props)
Modified:
nutch/site/forrest/src/documentation/content/xdocs/site.xml
nutch/site/publish/about.html
nutch/site/publish/about.pdf
nutch/site/publish/bot.html
nutch/site/publish/bot.pdf
nutch/site/publish/credits.html
nutch/site/publish/credits.pdf
nutch/site/publish/faq.html
nutch/site/publish/faq.pdf
nutch/site/publish/index.html
nutch/site/publish/index.pdf
nutch/site/publish/issue_tracking.html
nutch/site/publish/issue_tracking.pdf
nutch/site/publish/linkmap.html
nutch/site/publish/linkmap.pdf
nutch/site/publish/mailing_lists.html
nutch/site/publish/mailing_lists.pdf
nutch/site/publish/nightly.html
nutch/site/publish/nightly.pdf
nutch/site/publish/old_downloads.html
nutch/site/publish/old_downloads.pdf
nutch/site/publish/sonar.html
nutch/site/publish/sonar.pdf
nutch/site/publish/tutorial.html
nutch/site/publish/tutorial.pdf
nutch/site/publish/version_control.html
nutch/site/publish/version_control.pdf
nutch/site/publish/wiki.html
nutch/site/publish/wiki.pdf
Added: nutch/site/forrest/src/documentation/content/xdocs/downloads.xml
URL: http://svn.apache.org/viewvc/nutch/site/forrest/src/documentation/content/xdocs/downloads.xml?rev=1475631&view=auto
==============================================================================
--- nutch/site/forrest/src/documentation/content/xdocs/downloads.xml (added)
+++ nutch/site/forrest/src/documentation/content/xdocs/downloads.xml Thu Apr 25 05:05:56 2013
@@ -0,0 +1,111 @@
+<?xml version="1.0"?>
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<document>
+
+ <header>
+ <title>Nutch Downloads</title>
+ </header>
+
+ <body>
+
+ <section>
+ <title>Download</title>
+
+ <p> Apache Nutch 2.1 (src-tar and src-zip only) and 1.6 (src-tar, src-zip, bin-tar
and bin-zip) are now available. See
+ the
+ <a href="http://apache.org/dist/nutch/2.1/CHANGES-2.1.txt">CHANGES-2.1.txt</a>,
and
+ <a href="http://apache.org/dist/nutch/1.6/CHANGES_1.6.txt">CHANGES_1.6.txt</a>
+ files for more information on the list of updates in these releases.
+ </p>
+ <p> All Apache Nutch distributions is distributed under the <a href="http://www.apache.org/licenses/LICENSE-2.0.html">Apache
License, version 2.0</a>.
+ </p>
+ <p>The link in the Mirrors column below should display a list of available mirrors
with a default selection based on your inferred location. If you do not see that page, try
a different browser. The checksum and signature are links to the originals on the main distribution
server.
+ </p>
+
+<table>
+ <caption>Downloads</caption>
+ <tr><th>Version</th> <th>Mirrors</th> <th>Checksum</th>
<th>Signature</th></tr>
+ <tr><td>Apache Nutch 2.1 (src.tar.gz)</td><td><a href="http://www.apache.org/dyn/closer.cgi/nutch/2.1/apache-nutch-2.1-src.tar.gz">
+ apache-nutch-2.1-src.tar.gz </a></td> <td><a href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.tar.gz.md5">
+ apache-nutch-2.1-src.tar.gz.md5</a> </td> <td><a href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.tar.gz.asc">
+ apache-nutch-2.1-src.tar.gz.asc</a> </td></tr>
+ <tr><td>Apache Nutch 2.1 (src.zip)</td><td><a href="http://www.apache.org/dyn/closer.cgi/nutch/2.1/apache-nutch-2.1-src.zip">
+ apache-nutch-2.1-src.zip</a></td><td><a href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.zip.md5">
+ apache-nutch-2.1-src.zip.md5</a></td><td><a href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.zip.asc">
+ apache-nutch-2.1-src.zip.asc</a> </td></tr>
+ <tr><td>Apache Nutch 1.6 (src.tar.gz)</td><td><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-src.tar.gz">
+ apache-nutch-1.6-src.tar.gz</a></td> <td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.tar.gz.md5">
+ apache-nutch-1.6-src.tar.gz.md5</a> </td> <td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.tar.gz.asc">
+ apache-nutch-1.6-src.tar.gz.asc</a> </td></tr>
+ <tr><td>Apache Nutch 1.6 (src.zip)</td><td><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-src.zip">
+ apache-nutch-1.6-src.zip</a></td><td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.zip.md5">
+ apache-nutch-1.6-src.zip.md5</a></td><td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.zip.asc">
+ apache-nutch-1.6-src.zip.asc</a> </td></tr>
+ <tr><td>Apache Nutch 1.6 (bin.tar.gz)</td><td><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-bin.tar.gz">
+ apache-nutch-1.6-bin.tar.gz</a></td> <td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.tar.gz.md5">
+ apache-nutch-1.6-bin.tar.gz.md5</a> </td> <td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.tar.gz.asc">
+ apache-nutch-1.6-bin.tar.gz.asc</a> </td></tr>
+ <tr><td>Apache Nutch 1.6 (bin.zip)</td><td><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-bin.zip">
+ apache-nutch-1.6-bin.zip</a></td><td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.zip.md5">
+ apache-nutch-1.6-bin.zip.md5</a></td><td><a href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.zip.asc">
+ apache-nutch-1.6-bin.zip.asc</a> </td></tr>
+</table>
+ </section>
+
+ <section>
+ <title>Verify Releases</title>
+ <p>It is essential that you verify the integrity of the downloaded files using
the PGP or MD5 signatures. Please read <a href="http://httpd.apache.org/dev/verification.html">Verifying
Apache HTTP Server Releases</a> for more information on why you should verify our releases.
+ We strongly recommend you verify your downloads with both PGP and MD5.</p>
+
+ <p><strong>PGP Signature</strong></p>
+ <p>The PGP signatures can be verified using PGP or GPG. First download the <a
href="http://www.apache.org/dist/nutch/KEYS">KEYS</a> as well as the asc signature
file for the relevant distribution. Make sure you get these files from the <a href="http://www.apache.org/dist/nutch/">main
distribution directory</a>, rather than from a mirror. Then verify the signatures using
</p>
+ <p><code> $ gpg --import KEYS </code></p>
+ <p><code> $ gpg --verify apache-nutch-X.Y.Z </code></p>
+ <p>The files in Apache Nurch 2.1 and 1.6 releases are signed by Lewis John McGibbney
(lewismc) C601BCA7 </p>
+
+ <p><strong>MD5 Signature</strong></p>
+ <p>Alternatively, you can verify the MD5 signature on the files. A unix program
called md5 or md5sum is included in many unix distributions.</p>
+ <p><code> $ md5sum apache-nutch-X.Y.Z</code></p>
+ <p><code> ... output should match the string in apache-nutch-X.Y.Z</code></p>
+
+ </section>
+
+ <section>
+ <title>Previous Releases</title>
+ <p>If you are looking for previous releases of Apache Nutch, have a look in the
<a href="old_downloads.html">old downloads</a> page, or alternatively for even
older releases check out the
+ <a href="http://archive.apache.org/dist/incubator/nutch/">Incubator archives</a>.
+ </p>
+
+ <p> Subscribe to
+ the <code>dev@</code> <a href="mailing_lists.html">mailing list</a>
if you want to
+ get notified about future release candidates and subsequent Nutch official releases.
+ </p>
+
+ <p>Apache Nutch releases are available under the Apache License, Version 2.0.
See
+ the NOTICE.txt file contained in each release artifact for applicable copyright
+ attribution notices.
+ </p>
+ </section>
+
+ </body>
+
+</document>
Modified: nutch/site/forrest/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/nutch/site/forrest/src/documentation/content/xdocs/site.xml?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/forrest/src/documentation/content/xdocs/site.xml (original)
+++ nutch/site/forrest/src/documentation/content/xdocs/site.xml Thu Apr 25 05:05:56 2013
@@ -53,7 +53,7 @@ See http://forrest.apache.org/docs/linki
</docs>
<resources label="Resources">
- <download label="Download" href="ext:release" />
+ <download label="Download" href="downloads.html" />
<nightly label="Nightly builds" href="nightly.html" />
<sonar label="Sonar Analysis" href="sonar.html" />
<contact label="Mailing Lists" href="mailing_lists.html" />
@@ -80,7 +80,6 @@ See http://forrest.apache.org/docs/linki
<tika href="http://tika.apache.org/"/>
<gora href="http://gora.apache.org"/>
<store href="http://www.cafepress.com/nutch/"/>
- <release href="http://www.apache.org/dyn/closer.cgi/nutch/"/>
<license href="http://www.apache.org/licenses/"/>
<sponsor href="http://www.apache.org/foundation/sponsorship.html" />
<thanks href="http://www.apache.org/foundation/thanks.html" />
Modified: nutch/site/publish/about.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/about.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/about.html (original)
+++ nutch/site/publish/about.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/about.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/about.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/bot.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/bot.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/bot.html (original)
+++ nutch/site/publish/bot.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/bot.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/bot.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/credits.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/credits.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/credits.html (original)
+++ nutch/site/publish/credits.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/credits.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/credits.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Added: nutch/site/publish/downloads.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/downloads.html?rev=1475631&view=auto
==============================================================================
--- nutch/site/publish/downloads.html (added)
+++ nutch/site/publish/downloads.html Thu Apr 25 05:05:56 2013
@@ -0,0 +1,414 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
+<meta content="Apache Forrest" name="Generator">
+<meta name="Forrest-version" content="0.9">
+<meta name="Forrest-skin-name" content="nutch">
+<title>Nutch Downloads</title>
+<link type="text/css" href="skin/basic.css" rel="stylesheet">
+<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet">
+<link media="print" type="text/css" href="skin/print.css" rel="stylesheet">
+<link type="text/css" href="skin/profile.css" rel="stylesheet">
+<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script
src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script
src="skin/fontsize.js" language="javascript" type="text/javascript"></script>
+<link rel="shortcut icon" href="images/favicon.ico">
+</head>
+<body onload="init()">
+<script type="text/javascript">ndeSetTextSize();</script>
+<div id="top">
+<!--+
+ |breadtrail
+ +-->
+<div class="breadtrail">
+<a href="http://www.apache.org/">Apache</a> > <a href="http://nutch.apache.org">Nutch</a>
> <a href="http://nutch.apache.org">Home</a><script src="skin/breadcrumbs.js"
language="JavaScript" type="text/javascript"></script>
+</div>
+<!--+
+ |header
+ +-->
+<div class="header">
+<!--+
+ |start group logo
+ +-->
+<div class="grouplogo">
+<a href="http://www.apache.org/"><img class="logoImage" alt="Apache" src="images/feather-small.gif"
title="Apache Software Foundation "></a>
+</div>
+<!--+
+ |end group logo
+ +-->
+<!--+
+ |start Project Logo
+ +-->
+<div class="projectlogo">
+<a href="http://nutch.apache.org/"><img class="logoImage" alt="Nutch" src="images/nutch_logo_tm.gif"
title="Open Source Web Search Software"></a>
+</div>
+<!--+
+ |end Project Logo
+ +-->
+<!--+
+ |start Search
+ +-->
+<div class="searchbox">
+<script type="text/javascript">
+ function selectProvider(form) {
+ provider = form.elements['searchProvider'].value;
+ if (provider == "any") {
+ if (Math.random() > 0.5) {
+ provider = "lucid";
+ } else {
+ provider = "sl";
+ }
+ }
+
+ if (provider == "lucid") {
+ form.action = "http://search.lucidimagination.com/p:nutch";
+ } else if (provider == "sl") {
+ form.action = "http://search-lucene.com/nutch";
+ }
+
+ days = 90; // cookie will be valid for 90 days
+ date = new Date();
+ date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000));
+ expires = "; expires=" + date.toGMTString();
+ document.cookie = "searchProvider=" + provider + expires + "; path=/";
+ }
+ </script>
+<form id="searchform" action="http://search.lucidimagination.com/p:nutch" method="get"
class="roundtopsmall">
+<input onFocus="getBlank (this, 'Search the site with Solr');" size="25" name="q" id="query"
type="text" value="Search the site with Solr">
+ <input onclick="selectProvider(this.form)" name="Search" value="Search"
type="submit">
+ @
+ <select id="searchProvider" name="searchProvider"><option
value="any">select provider</option><option value="lucid">Lucid Find</option><option
value="sl">Search-Lucene</option></select><script type="text/javascript">
+ if (document.cookie.length>0) {
+ cStart=document.cookie.indexOf("searchProvider=");
+ if (cStart!=-1) {
+ cStart=cStart + "searchProvider=".length;
+ cEnd=document.cookie.indexOf(";", cStart);
+ if (cEnd==-1) {
+ cEnd=document.cookie.length;
+ }
+ provider = unescape(document.cookie.substring(cStart,cEnd));
+ document.forms['searchform'].elements['searchProvider'].value
= provider;
+ }
+ }
+ </script>
+</form>
+</div>
+<!--+
+ |end search
+ +-->
+<!--+
+ |start Tabs
+ +-->
+<ul id="tabs">
+<li class="current">
+<a class="selected" href="index.html">Main</a>
+</li>
+<li>
+<a class="unselected" href="wiki.html">Wiki</a>
+</li>
+<li>
+<a class="unselected" href="http://issues.apache.org/jira/browse/NUTCH">Jira</a>
+</li>
+</ul>
+<!--+
+ |end Tabs
+ +-->
+</div>
+</div>
+<div id="main">
+<div id="publishedStrip">
+<!--+
+ |start Subtabs
+ +-->
+<div id="level2tabs"></div>
+<!--+
+ |end Endtabs
+ +-->
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+// --></script>
+</div>
+<!--+
+ |breadtrail
+ +-->
+<div class="breadtrail">
+
+
+ </div>
+<!--+
+ |start Menu, mainarea
+ +-->
+<!--+
+ |start Menu
+ +-->
+<div id="menu">
+<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Project</div>
+<div id="menu_1.1" class="menuitemgroup">
+<div class="menuitem">
+<a href="index.html">News</a>
+</div>
+<div class="menuitem">
+<a href="about.html">About</a>
+</div>
+<div class="menuitem">
+<a href="credits.html">Credits</a>
+</div>
+<div class="menuitem">
+<a href="http://www.apache.org/foundation/thanks.html">Thanks</a>
+</div>
+<div class="menuitem">
+<a href="http://www.cafepress.com/nutch/">Buy Stuff</a>
+</div>
+<div class="menuitem">
+<a href="http://www.apache.org/foundation/sponsorship.html">Sponsorship</a>
+</div>
+<div class="menuitem">
+<a href="http://www.apache.org/licenses/">License</a>
+</div>
+<div class="menuitem">
+<a href="http://www.apache.org/security/">Security</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Documentation</div>
+<div id="menu_1.2" class="menuitemgroup">
+<div class="menuitem">
+<a href="faq.html">FAQ</a>
+</div>
+<div class="menuitem">
+<a href="wiki.html">Wiki</a>
+</div>
+<div class="menuitem">
+<a href="tutorial.html">Tutorial</a>
+</div>
+<div class="menuitem">
+<a href="bot.html">Robot </a>
+</div>
+<div class="menuitem">
+<a href="apidocs-2.1/index.html">API Docs (2.1)</a>
+</div>
+<div class="menuitem">
+<a href="apidocs-1.6/index.html">API Docs (1.6)</a>
+</div>
+<div class="menuitem">
+<a href="https://builds.apache.org/job/Nutch-trunk/javadoc/">API Docs (trunk nightly)</a>
+</div>
+<div class="menuitem">
+<a href="https://builds.apache.org/job/Nutch-nutchgora/javadoc/">API Docs (2.x nightly)</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
+<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
+<div class="menupage">
+<div class="menupagetitle">Download</div>
+</div>
+<div class="menuitem">
+<a href="nightly.html">Nightly builds</a>
+</div>
+<div class="menuitem">
+<a href="sonar.html">Sonar Analysis</a>
+</div>
+<div class="menuitem">
+<a href="mailing_lists.html">Mailing Lists</a>
+</div>
+<div class="menuitem">
+<a href="issue_tracking.html">Issue Tracking</a>
+</div>
+<div class="menuitem">
+<a href="version_control.html">Version Control</a>
+</div>
+<div class="menuitem">
+<a href="old_downloads.html">Older Downloads</a>
+</div>
+</div>
+<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">Related
Projects</div>
+<div id="menu_1.4" class="menuitemgroup">
+<div class="menuitem">
+<a href="http://lucene.apache.org/java/">Lucene</a>
+</div>
+<div class="menuitem">
+<a href="http://hadoop.apache.org/">Hadoop</a>
+</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/solr/">Solr</a>
+</div>
+<div class="menuitem">
+<a href="http://tika.apache.org/">Tika</a>
+</div>
+<div class="menuitem">
+<a href="http://gora.apache.org">Gora</a>
+</div>
+</div>
+<div id="credit"></div>
+<div id="roundbottom">
+<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div>
+<!--+
+ |alternative credits
+ +-->
+<div id="credit2"></div>
+</div>
+<!--+
+ |end Menu
+ +-->
+<!--+
+ |start content
+ +-->
+<div id="content">
+<div title="Portable Document Format" class="pdflink">
+<a class="dida" href="downloads.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif"
class="skin"><br>
+ PDF</a>
+</div>
+<h1>Nutch Downloads</h1>
+<div id="minitoc-area">
+<ul class="minitoc">
+<li>
+<a href="#Download">Download</a>
+</li>
+<li>
+<a href="#Verify+Releases">Verify Releases</a>
+</li>
+<li>
+<a href="#Previous+Releases">Previous Releases</a>
+</li>
+</ul>
+</div>
+
+
+<a name="N1000E"></a><a name="Download"></a>
+<h2 class="h3">Download</h2>
+<div class="section">
+<p> Apache Nutch 2.1 (src-tar and src-zip only) and 1.6 (src-tar, src-zip, bin-tar
and bin-zip) are now available. See
+ the
+ <a href="http://apache.org/dist/nutch/2.1/CHANGES-2.1.txt">CHANGES-2.1.txt</a>,
and
+ <a href="http://apache.org/dist/nutch/1.6/CHANGES_1.6.txt">CHANGES_1.6.txt</a>
+ files for more information on the list of updates in these releases.
+ </p>
+<p> All Apache Nutch distributions is distributed under the <a href="http://www.apache.org/licenses/LICENSE-2.0.html">Apache
License, version 2.0</a>.
+ </p>
+<p>The link in the Mirrors column below should display a list of available mirrors
with a default selection based on your inferred location. If you do not see that page, try
a different browser. The checksum and signature are links to the originals on the main distribution
server.
+ </p>
+<table class="ForrestTable" cellspacing="1" cellpadding="4">
+
+<caption>Downloads</caption>
+
+<tr>
+<th colspan="1" rowspan="1">Version</th> <th colspan="1" rowspan="1">Mirrors</th>
<th colspan="1" rowspan="1">Checksum</th> <th colspan="1" rowspan="1">Signature</th>
+</tr>
+
+<tr>
+<td colspan="1" rowspan="1">Apache Nutch 2.1 (src.tar.gz)</td><td colspan="1"
rowspan="1"><a href="http://www.apache.org/dyn/closer.cgi/nutch/2.1/apache-nutch-2.1-src.tar.gz">
+ apache-nutch-2.1-src.tar.gz </a></td> <td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.tar.gz.md5">
+ apache-nutch-2.1-src.tar.gz.md5</a> </td> <td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.tar.gz.asc">
+ apache-nutch-2.1-src.tar.gz.asc</a> </td>
+</tr>
+
+<tr>
+<td colspan="1" rowspan="1">Apache Nutch 2.1 (src.zip)</td><td colspan="1"
rowspan="1"><a href="http://www.apache.org/dyn/closer.cgi/nutch/2.1/apache-nutch-2.1-src.zip">
+ apache-nutch-2.1-src.zip</a></td><td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.zip.md5">
+ apache-nutch-2.1-src.zip.md5</a></td><td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/2.1/apache-nutch-2.1-src.zip.asc">
+ apache-nutch-2.1-src.zip.asc</a> </td>
+</tr>
+
+<tr>
+<td colspan="1" rowspan="1">Apache Nutch 1.6 (src.tar.gz)</td><td colspan="1"
rowspan="1"><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-src.tar.gz">
+ apache-nutch-1.6-src.tar.gz</a></td> <td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.tar.gz.md5">
+ apache-nutch-1.6-src.tar.gz.md5</a> </td> <td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.tar.gz.asc">
+ apache-nutch-1.6-src.tar.gz.asc</a> </td>
+</tr>
+
+<tr>
+<td colspan="1" rowspan="1">Apache Nutch 1.6 (src.zip)</td><td colspan="1"
rowspan="1"><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-src.zip">
+ apache-nutch-1.6-src.zip</a></td><td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.zip.md5">
+ apache-nutch-1.6-src.zip.md5</a></td><td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-src.zip.asc">
+ apache-nutch-1.6-src.zip.asc</a> </td>
+</tr>
+
+<tr>
+<td colspan="1" rowspan="1">Apache Nutch 1.6 (bin.tar.gz)</td><td colspan="1"
rowspan="1"><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-bin.tar.gz">
+ apache-nutch-1.6-bin.tar.gz</a></td> <td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.tar.gz.md5">
+ apache-nutch-1.6-bin.tar.gz.md5</a> </td> <td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.tar.gz.asc">
+ apache-nutch-1.6-bin.tar.gz.asc</a> </td>
+</tr>
+
+<tr>
+<td colspan="1" rowspan="1">Apache Nutch 1.6 (bin.zip)</td><td colspan="1"
rowspan="1"><a href="http://www.apache.org/dyn/closer.cgi/nutch/1.6/apache-nutch-1.6-bin.zip">
+ apache-nutch-1.6-bin.zip</a></td><td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.zip.md5">
+ apache-nutch-1.6-bin.zip.md5</a></td><td colspan="1" rowspan="1"><a
href="http://apache.org/dist/nutch/1.6/apache-nutch-1.6-bin.zip.asc">
+ apache-nutch-1.6-bin.zip.asc</a> </td>
+</tr>
+
+</table>
+</div>
+
+
+<a name="N100E4"></a><a name="Verify+Releases"></a>
+<h2 class="h3">Verify Releases</h2>
+<div class="section">
+<p>It is essential that you verify the integrity of the downloaded files using the
PGP or MD5 signatures. Please read <a href="http://httpd.apache.org/dev/verification.html">Verifying
Apache HTTP Server Releases</a> for more information on why you should verify our releases.
+ We strongly recommend you verify your downloads with both PGP and MD5.</p>
+<p>
+<strong>PGP Signature</strong>
+</p>
+<p>The PGP signatures can be verified using PGP or GPG. First download the <a href="http://www.apache.org/dist/nutch/KEYS">KEYS</a>
as well as the asc signature file for the relevant distribution. Make sure you get these files
from the <a href="http://www.apache.org/dist/nutch/">main distribution directory</a>,
rather than from a mirror. Then verify the signatures using </p>
+<p>
+<span class="codefrag"> $ gpg --import KEYS </span>
+</p>
+<p>
+<span class="codefrag"> $ gpg --verify apache-nutch-X.Y.Z </span>
+</p>
+<p>The files in Apache Nurch 2.1 and 1.6 releases are signed by Lewis John McGibbney
(lewismc) C601BCA7 </p>
+<p>
+<strong>MD5 Signature</strong>
+</p>
+<p>Alternatively, you can verify the MD5 signature on the files. A unix program called
md5 or md5sum is included in many unix distributions.</p>
+<p>
+<span class="codefrag"> $ md5sum apache-nutch-X.Y.Z</span>
+</p>
+<p>
+<span class="codefrag"> ... output should match the string in apache-nutch-X.Y.Z</span>
+</p>
+</div>
+
+
+<a name="N1011B"></a><a name="Previous+Releases"></a>
+<h2 class="h3">Previous Releases</h2>
+<div class="section">
+<p>If you are looking for previous releases of Apache Nutch, have a look in the <a
href="old_downloads.html">old downloads</a> page, or alternatively for even older
releases check out the
+ <a href="http://archive.apache.org/dist/incubator/nutch/">Incubator archives</a>.
+ </p>
+<p> Subscribe to
+ the <span class="codefrag">dev@</span> <a href="mailing_lists.html">mailing
list</a> if you want to
+ get notified about future release candidates and subsequent Nutch official releases.
+ </p>
+<p>Apache Nutch releases are available under the Apache License, Version 2.0. See
+ the NOTICE.txt file contained in each release artifact for applicable copyright
+ attribution notices.
+ </p>
+</div>
+
+
+</div>
+<!--+
+ |end content
+ +-->
+<div class="clearboth"> </div>
+</div>
+<div id="footer">
+<!--+
+ |start bottomstrip
+ +-->
+<div class="lastmodified">
+<script type="text/javascript"><!--
+document.write("Last Published: " + document.lastModified);
+// --></script>
+</div>
+<div class="copyright">
+ Copyright ©
+ 2005-2011 <a href="http://www.apache.org/licenses/">The Apache Software Foundation.
+Apache Nutch, Nutch, Apache, the Apache feather logo, and the Apache Nutch project logo are
trademarks of The Apache Software Foundation.
+ </a>
+</div>
+<!--+
+ |end bottomstrip
+ +-->
+</div>
+</body>
+</html>
Added: nutch/site/publish/downloads.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/downloads.pdf?rev=1475631&view=auto
==============================================================================
Binary file - no diff available.
Propchange: nutch/site/publish/downloads.pdf
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: nutch/site/publish/faq.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/faq.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/faq.html (original)
+++ nutch/site/publish/faq.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/faq.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/faq.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/index.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/index.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/index.html (original)
+++ nutch/site/publish/index.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/index.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/index.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/issue_tracking.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/issue_tracking.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/issue_tracking.html (original)
+++ nutch/site/publish/issue_tracking.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/issue_tracking.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/issue_tracking.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/linkmap.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/linkmap.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/linkmap.html (original)
+++ nutch/site/publish/linkmap.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
@@ -391,7 +391,7 @@ document.write("Last Published: " + docu
<ul>
<li>
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a> ___________________ <em>download</em>
+<a href="downloads.html">Download</a> ___________________ <em>download</em>
</li>
</ul>
Modified: nutch/site/publish/linkmap.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/linkmap.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/mailing_lists.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/mailing_lists.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/mailing_lists.html (original)
+++ nutch/site/publish/mailing_lists.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/mailing_lists.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/mailing_lists.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/nightly.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/nightly.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/nightly.html (original)
+++ nutch/site/publish/nightly.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menupage">
<div class="menupagetitle">Nightly builds</div>
Modified: nutch/site/publish/nightly.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/nightly.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/old_downloads.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/old_downloads.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/old_downloads.html (original)
+++ nutch/site/publish/old_downloads.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/old_downloads.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/old_downloads.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Added: nutch/site/publish/skin/images/apache-thanks.png
URL: http://svn.apache.org/viewvc/nutch/site/publish/skin/images/apache-thanks.png?rev=1475631&view=auto
==============================================================================
Binary file - no diff available.
Propchange: nutch/site/publish/skin/images/apache-thanks.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: nutch/site/publish/skin/images/built-with-cocoon.gif
URL: http://svn.apache.org/viewvc/nutch/site/publish/skin/images/built-with-cocoon.gif?rev=1475631&view=auto
==============================================================================
Binary file - no diff available.
Propchange: nutch/site/publish/skin/images/built-with-cocoon.gif
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: nutch/site/publish/sonar.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/sonar.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/sonar.html (original)
+++ nutch/site/publish/sonar.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/sonar.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/sonar.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/tutorial.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/tutorial.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/tutorial.html (original)
+++ nutch/site/publish/tutorial.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/tutorial.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/tutorial.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/version_control.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/version_control.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/version_control.html (original)
+++ nutch/site/publish/version_control.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle"
style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/version_control.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/version_control.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Modified: nutch/site/publish/wiki.html
URL: http://svn.apache.org/viewvc/nutch/site/publish/wiki.html?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
--- nutch/site/publish/wiki.html (original)
+++ nutch/site/publish/wiki.html Thu Apr 25 05:05:56 2013
@@ -198,7 +198,7 @@ document.write("Last Published: " + docu
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
<div class="menuitem">
-<a href="http://www.apache.org/dyn/closer.cgi/nutch/">Download</a>
+<a href="downloads.html">Download</a>
</div>
<div class="menuitem">
<a href="nightly.html">Nightly builds</a>
Modified: nutch/site/publish/wiki.pdf
URL: http://svn.apache.org/viewvc/nutch/site/publish/wiki.pdf?rev=1475631&r1=1475630&r2=1475631&view=diff
==============================================================================
Binary files - no diff available.
Author: jnioche
Date: Thu Apr 18 09:26:07 2013
New Revision: 1469241
URL: http://svn.apache.org/r1469241
Log:
Fixed detection of job file for distributed mode
Modified:
nutch/branches/2.x/src/bin/crawl
Modified: nutch/branches/2.x/src/bin/crawl
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/bin/crawl?rev=1469241&r1=1469240&r2=1469241&view=diff
==============================================================================
--- nutch/branches/2.x/src/bin/crawl (original)
+++ nutch/branches/2.x/src/bin/crawl Thu Apr 18 09:26:07 2013
@@ -73,7 +73,7 @@ addDays=0
# determines whether mode based on presence of job file
mode=local
-if [ -f ../nutch-*.job ]; then
+if [ -f ../*nutch-*.job ]; then
mode=distributed
fi
Author: jnioche
Date: Thu Apr 18 09:08:16 2013
New Revision: 1469231
URL: http://svn.apache.org/r1469231
Log:
Fixed detection of job file for distributed mode
Modified:
nutch/trunk/src/bin/crawl
Modified: nutch/trunk/src/bin/crawl
URL: http://svn.apache.org/viewvc/nutch/trunk/src/bin/crawl?rev=1469231&r1=1469230&r2=1469231&view=diff
==============================================================================
--- nutch/trunk/src/bin/crawl (original)
+++ nutch/trunk/src/bin/crawl Thu Apr 18 09:08:16 2013
@@ -72,7 +72,7 @@ numThreads=50
# determines whether mode based on presence of job file
mode=local
-if [ -f ../nutch-*.job ]; then
+if [ -f ../*nutch-*.job ]; then
mode=distributed
fi
Author: lewismc
Date: Wed Apr 17 23:22:59 2013
New Revision: 1469100
URL: http://svn.apache.org/r1469100
Log:
NUTCH-1501 Harmonize behavior of parsechecker and indexchecker
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1469100&r1=1469099&r2=1469100&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Wed Apr 17 23:22:59 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1501 Harmonize behavior of parsechecker and indexchecker (snagel + lewismc)
+
* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
Modified: nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java?rev=1469100&r1=1469099&r2=1469100&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java Wed Apr 17 23:22:59
2013
@@ -68,7 +68,7 @@ public class IndexingFiltersChecker exte
if (args.length != 1) {
System.err.println(usage);
- System.exit(-1);
+ return -1;
}
url = URLUtil.toASCII(args[0]);
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1469100&r1=1469099&r2=1469100&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java Wed Apr 17 23:22:59 2013
@@ -35,19 +35,38 @@ import org.apache.nutch.util.StringUtil;
/**
* Parser checker, useful for testing parser.
- *
+ * It also accurately reports possible fetching and
+ * parsing failures and presents protocol status signals to aid
+ * debugging. The tool enables us to retrieve the following data from
+ * any url:
+ * <ol>
+ * <li><tt>contentType</tt>: The URL {@link org.apache.nutch.protocol.Content}
type.</li>
+ * <li><tt>signature</tt>: Digest is used to identify pages (like unique
ID) and is used to remove
+ * duplicates during the dedup procedure.
+ * It is calculated using {@link org.apache.nutch.crawl.MD5Signature} or
+ * {@link org.apache.nutch.crawl.TextProfileSignature}.</li>
+ * <li><tt>Version</tt>: From {@link org.apache.nutch.parse.ParseData}.</li>
+ * <li><tt>Status</tt>: From {@link org.apache.nutch.parse.ParseData}.</li>
+ * <li><tt>Title</tt>: of the URL</li>
+ * <li><tt>Outlinks</tt>: associated with the URL</li>
+ * <li><tt>Content Metadata</tt>: such as <i>X-AspNet-Version</i>,
<i>Date</i>,
+ * <i>Content-length</i>, <i>servedBy</i>, <i>Content-Type</i>,
<i>Cache-Control</>, etc.</li>
+ * <li><tt>Parse Metadata</tt>: such as <i>CharEncodingForConversion</i>,
+ * <i>OriginalCharEncoding</i>, <i>language</i>, etc.</li>
+ * <li><tt>ParseText</tt>: The page parse text which varies in length depdnecing
on
+ * <code>content.length</code> configuration.</li>
+ * </ol>
* @author John Xing
*/
public class ParserChecker implements Tool {
public static final Logger LOG = LoggerFactory.getLogger(ParserChecker.class);
+ private Configuration conf;
public ParserChecker() {
}
- Configuration conf = null;
-
public int run(String[] args) throws Exception {
boolean dumpText = false;
boolean force = false;
@@ -57,8 +76,8 @@ public class ParserChecker implements To
String usage = "Usage: ParserChecker [-dumpText] [-forceAs mimeType] url";
if (args.length == 0) {
- System.err.println(usage);
- System.exit(-1);
+ LOG.error(usage);
+ return (-1);
}
for (int i = 0; i < args.length; i++) {
@@ -68,7 +87,7 @@ public class ParserChecker implements To
} else if (args[i].equals("-dumpText")) {
dumpText = true;
} else if (i != args.length - 1) {
- System.err.println(usage);
+ LOG.error(usage);
System.exit(-1);
} else {
url = URLUtil.toASCII(args[i]);
@@ -102,7 +121,7 @@ public class ParserChecker implements To
}
if (contentType == null) {
- System.err.println("");
+ LOG.error("Failed to determine content type!");
return (-1);
}
@@ -112,9 +131,14 @@ public class ParserChecker implements To
ParseResult parseResult = new ParseUtil(conf).parse(content);
+ if (parseResult == null) {
+ LOG.error("Problem with parse - check log");
+ return (-1);
+ }
+
// Calculate the signature
byte[] signature = SignatureFactory.getSignature(getConf()).calculate(content, parseResult.get(new
Text(url)));
-
+
if (LOG.isInfoEnabled()) {
LOG.info("parsing: " + url);
LOG.info("contentType: " + contentType);
@@ -123,12 +147,12 @@ public class ParserChecker implements To
for (java.util.Map.Entry<Text, Parse> entry : parseResult) {
Parse parse = entry.getValue();
- System.out.print("---------\nUrl\n---------------\n");
+ LOG.info("---------\nUrl\n---------------\n");
System.out.print(entry.getKey());
- System.out.print("\n---------\nParseData\n---------\n");
+ LOG.info("\n---------\nParseData\n---------\n");
System.out.print(parse.getData().toString());
if (dumpText) {
- System.out.print("---------\nParseText\n---------\n");
+ LOG.info("---------\nParseText\n---------\n");
System.out.print(parse.getText());
}
}
Author: lewismc
Date: Wed Apr 17 23:20:36 2013
New Revision: 1469099
URL: http://svn.apache.org/r1469099
Log:
NUTCH-1501 Harmonize behavior of parsechecker and indexchecker
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/conf/log4j.properties
nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java
nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Wed Apr 17 23:20:36 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1501 Harmonize behavior of parsechecker and indexchecker (snagel + lewismc)
+
* NUTCH-1551 Improve WebTableReader field order and display batchId (lewismc)
* NUTCH-1552 possibility of a NPE in index-more plugin (kaveh minooie via lewismc)
Modified: nutch/branches/2.x/conf/log4j.properties
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/conf/log4j.properties?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/conf/log4j.properties (original)
+++ nutch/branches/2.x/conf/log4j.properties Wed Apr 17 23:20:36 2013
@@ -38,6 +38,8 @@ log4j.logger.org.apache.nutch.indexer.so
log4j.logger.org.apache.nutch.indexer.DeleteDuplicates=INFO,cmdstdout
log4j.logger.org.apache.nutch.crawl.WebTableReader=INFO,cmdstdout
log4j.logger.org.apache.nutch.host.HostDbReader=INFO,cmdstdout
+log4j.logger.org.apache.nutch.parse.ParserChecker=INFO,cmdstdout
+log4j.logger.org.apache.nutch.indexer.IndexingFiltersChecker=INFO,cmdstdout
log4j.logger.org.apache.nutch=INFO
log4j.logger.org.apache.hadoop=WARN
Modified: nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java Wed Apr
17 23:20:36 2013
@@ -37,6 +37,7 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolStatusUtils;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.URLUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,7 +68,7 @@ public class IndexingFiltersChecker exte
return -1;
}
- url = args[0];
+ url = URLUtil.toASCII(args[0]);
if (LOG.isInfoEnabled()) {
LOG.info("fetching: " + url);
Modified: nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java Wed Apr 17 23:20:36
2013
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.SignatureFactory;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolFactory;
@@ -36,10 +37,32 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.util.Bytes;
import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.StringUtil;
+import org.apache.nutch.util.URLUtil;
/**
* Parser checker, useful for testing parser.
- *
+ * It also accurately reports possible fetching and
+ * parsing failures and presents protocol status signals to aid
+ * debugging. The tool enables us to retrieve the following data from
+ * any url:
+ * <ol>
+ * <li><tt>contentType</tt>: The URL {@link org.apache.nutch.protocol.Content}
type.</li>
+ * <li><tt>signature</tt>: Digest is used to identify pages (like unique
ID) and is used to remove
+ * duplicates during the dedup procedure.
+ * It is calculated using {@link org.apache.nutch.crawl.MD5Signature} or
+ * {@link org.apache.nutch.crawl.TextProfileSignature}.</li>
+ * <li><tt>Version</tt>: From {@link org.apache.nutch.parse.ParseData}.</li>
+ * <li><tt>Status</tt>: From {@link org.apache.nutch.parse.ParseData}.</li>
+ * <li><tt>Title</tt>: of the URL</li>
+ * <li><tt>Outlinks</tt>: associated with the URL</li>
+ * <li><tt>Content Metadata</tt>: such as <i>X-AspNet-Version</i>,
<i>Date</i>,
+ * <i>Content-length</i>, <i>servedBy</i>, <i>Content-Type</i>,
<i>Cache-Control</>, etc.</li>
+ * <li><tt>Parse Metadata</tt>: such as <i>CharEncodingForConversion</i>,
+ * <i>OriginalCharEncoding</i>, <i>language</i>, etc.</li>
+ * <li><tt>ParseText</tt>: The page parse text which varies in length depdnecing
on
+ * <code>content.length</code> configuration.</li>
+ * </ol>
* @author John Xing
*/
@@ -60,7 +83,7 @@ public class ParserChecker implements To
String usage = "Usage: ParserChecker [-dumpText] [-forceAs mimeType] url";
if (args.length == 0) {
- System.err.println(usage);
+ LOG.error(usage);
return (-1);
}
@@ -71,10 +94,10 @@ public class ParserChecker implements To
} else if (args[i].equals("-dumpText")) {
dumpText = true;
} else if (i != args.length - 1) {
- System.err.println(usage);
+ LOG.error(usage);
System.exit(-1);
} else {
- url = args[i];
+ url = URLUtil.toASCII(args[i]);
}
}
@@ -110,15 +133,10 @@ public class ParserChecker implements To
}
if (contentType == null) {
- System.err.println("");
+ LOG.error("Failed to determine content type!");
return (-1);
}
- if (LOG.isInfoEnabled()) {
- LOG.info("parsing: " + url);
- LOG.info("contentType: " + contentType);
- }
-
page.setContentType(new Utf8(contentType));
if (ParserJob.isTruncated(url, page)) {
@@ -128,13 +146,23 @@ public class ParserChecker implements To
Parse parse = new ParseUtil(conf).parse(url, page);
if (parse == null) {
- System.err.println("Problem with parse - check log");
+ LOG.error("Problem with parse - check log");
return (-1);
}
+
+ // Calculate the signature
+ byte[] signature = SignatureFactory.getSignature(getConf()).calculate(page);
+
+ if (LOG.isInfoEnabled()) {
+ LOG.info("parsing: " + url);
+ LOG.info("contentType: " + contentType);
+ LOG.info("signature: " + StringUtil.toHexString(signature));
+ }
+
- System.out.print("---------\nUrl\n---------------\n");
+ LOG.info("---------\nUrl\n---------------\n");
System.out.print(url + "\n");
- System.out.print("---------\nMetadata\n---------\n");
+ LOG.info("---------\nMetadata\n---------\n");
Map<Utf8, ByteBuffer> metadata = page.getMetadata();
StringBuffer sb = new StringBuffer();
if (metadata != null) {
@@ -148,7 +176,7 @@ public class ParserChecker implements To
System.out.print(sb.toString());
}
if (dumpText) {
- System.out.print("---------\nParseText\n---------\n");
+ LOG.info("---------\nParseText\n---------\n");
System.out.print(parse.getText());
}
@@ -170,4 +198,5 @@ public class ParserChecker implements To
args);
System.exit(res);
}
+
}
Modified: nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java?rev=1469099&r1=1469098&r2=1469099&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/util/URLUtil.java Wed Apr 17 23:20:36 2013
@@ -18,7 +18,7 @@
package org.apache.nutch.util;
import java.net.MalformedURLException;
-import java.net.URL;
+import java.net.*;
import java.util.regex.Pattern;
import org.apache.nutch.util.domain.DomainSuffix;
@@ -333,6 +333,43 @@ public class URLUtil {
}
}
+ public static String toASCII(String url) {
+ try {
+ URL u = new URL(url);
+ URI p = new URI(u.getProtocol(),
+ null,
+ IDN.toASCII(u.getHost()),
+ u.getPort(),
+ u.getPath(),
+ u.getQuery(),
+ u.getRef());
+
+ return p.toString();
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+
+ public static String toUNICODE(String url) {
+ try {
+ URL u = new URL(url);
+ URI p = new URI(u.getProtocol(),
+ null,
+ IDN.toUnicode(u.getHost()),
+ u.getPort(),
+ u.getPath(),
+ u.getQuery(),
+ u.getRef());
+
+ return p.toString();
+ }
+ catch (Exception e) {
+ return null;
+ }
+ }
+
+
/** For testing */
public static void main(String[] args){
Author: lewismc
Date: Mon Apr 8 23:53:53 2013
New Revision: 1465834
URL: http://svn.apache.org/r1465834
Log:
revert NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1465834&r1=1465833&r2=1465834&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Mon Apr 8 23:53:53 2013
@@ -2,8 +2,6 @@ Nutch Change Log
Release 2.2 - Current Development
-* NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
(lewismc)
-
* NUTCH-1551 Improve WebTableReader field order and display batchId (lewismc)
* NUTCH-1552 possibility of a NPE in index-more plugin (kaveh minooie via lewismc)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java?rev=1465834&r1=1465833&r2=1465834&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java Mon Apr
8 23:53:53 2013
@@ -34,7 +34,7 @@ import java.text.ParseException;
public class HttpDateFormat {
protected static SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
/**
* HTTP date uses TimeZone GMT
Author: lewismc
Date: Mon Apr 8 23:52:24 2013
New Revision: 1465831
URL: http://svn.apache.org/r1465831
Log:
revert NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1465831&r1=1465830&r2=1465831&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Mon Apr 8 23:52:24 2013
@@ -2,8 +2,6 @@ Nutch Change Log
(trunk): Current Development
-* NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
(lewismc)
-
* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
Modified: nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java?rev=1465831&r1=1465830&r2=1465831&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java Mon Apr 8 23:52:24
2013
@@ -34,7 +34,7 @@ import java.text.ParseException;
public class HttpDateFormat {
protected static SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
/**
* HTTP date uses TimeZone GMT
Author: lewismc
Date: Mon Apr 8 19:46:29 2013
New Revision: 1465741
URL: http://svn.apache.org/r1465741
Log:
NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1465741&r1=1465740&r2=1465741&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Mon Apr 8 19:46:29 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
(lewismc)
+
* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
Modified: nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java?rev=1465741&r1=1465740&r2=1465741&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java Mon Apr 8 19:46:29
2013
@@ -34,7 +34,7 @@ import java.text.ParseException;
public class HttpDateFormat {
protected static SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
/**
* HTTP date uses TimeZone GMT
Author: lewismc
Date: Mon Apr 8 19:46:29 2013
New Revision: 1465742
URL: http://svn.apache.org/r1465742
Log:
NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1465742&r1=1465741&r2=1465742&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Mon Apr 8 19:46:29 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1554 org.apache.nutch.net.protocols.HttpDateFormat should NOT be Locale.US aware
(lewismc)
+
* NUTCH-1551 Improve WebTableReader field order and display batchId (lewismc)
* NUTCH-1552 possibility of a NPE in index-more plugin (kaveh minooie via lewismc)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java?rev=1465742&r1=1465741&r2=1465742&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/net/protocols/HttpDateFormat.java Mon Apr
8 19:46:29 2013
@@ -34,7 +34,7 @@ import java.text.ParseException;
public class HttpDateFormat {
protected static SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz", Locale.US);
+ new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss zzz");
/**
* HTTP date uses TimeZone GMT
Author: lewismc
Date: Mon Apr 8 00:36:59 2013
New Revision: 1465522
URL: http://svn.apache.org/r1465522
Log:
NUTCH-1532 Replace 'segment' mapping field with batchId
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexUtil.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1465522&r1=1465521&r2=1465522&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Mon Apr 8 00:36:59 2013
@@ -14,7 +14,7 @@ Release 2.2 - Current Development
* NUTCH-1038 Port IndexingFiltersChecker to 2.0 (snagel via lewismc)
-* NUTCH-1532 Replace 'segment' mapping field with batchId (Feng +via lewismc)
+* NUTCH-1532 Replace 'segment' mapping field with batchId (patches v2 + v3) (Feng +via lewismc)
* NUTCH-1533 Implement getPrevModifiedTime(), setPrevModifiedTime(), getBatchId() and setBatchId()
accessors in o.a.n.storage.WebPage (Feng via lewismc)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexUtil.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexUtil.java?rev=1465522&r1=1465521&r2=1465522&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexUtil.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexUtil.java Mon Apr 8 00:36:59
2013
@@ -63,7 +63,9 @@ public class IndexUtil {
NutchDocument doc = new NutchDocument();
doc.add("id", key);
doc.add("digest", StringUtil.toHexString(page.getSignature().array()));
- doc.add("batchId", page.getBatchId().toString());
+ if (page.getBatchId() != null) {
+ doc.add("batchId", page.getBatchId().toString());
+ }
String url = TableUtil.unreverseUrl(key);
Author: lewismc
Date: Mon Apr 8 00:33:23 2013
New Revision: 1465521
URL: http://svn.apache.org/r1465521
Log:
NUTCH-1551 Improve WebTableReader field order and display batchId
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/crawl/WebTableReader.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1465521&r1=1465520&r2=1465521&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Mon Apr 8 00:33:23 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1551 Improve WebTableReader field order and display batchId (lewismc)
+
* NUTCH-1552 possibility of a NPE in index-more plugin (kaveh minooie via lewismc)
* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/crawl/WebTableReader.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/crawl/WebTableReader.java?rev=1465521&r1=1465520&r2=1465521&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/crawl/WebTableReader.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/crawl/WebTableReader.java Mon Apr 8 00:33:23
2013
@@ -341,24 +341,34 @@ public class WebTableReader extends Nutc
sb.append("baseUrl:\t" + page.getBaseUrl()).append("\n");
sb.append("status:\t").append(page.getStatus()).append(" (").append(
CrawlStatus.getName((byte) page.getStatus())).append(")\n");
- sb.append("fetchInterval:\t" + page.getFetchInterval()).append("\n");
sb.append("fetchTime:\t" + page.getFetchTime()).append("\n");
sb.append("prevFetchTime:\t" + page.getPrevFetchTime()).append("\n");
- sb.append("retries:\t" + page.getRetriesSinceFetch()).append("\n");
+ sb.append("fetchInterval:\t" + page.getFetchInterval()).append("\n");
+ sb.append("retriesSinceFetch:\t" + page.getRetriesSinceFetch()).append("\n");
sb.append("modifiedTime:\t" + page.getModifiedTime()).append("\n");
+ sb.append("prevModifiedTime:\t" + page.getPrevModifiedTime()).append("\n");
sb.append("protocolStatus:\t" +
ProtocolStatusUtils.toString(page.getProtocolStatus())).append("\n");
- sb.append("parseStatus:\t" +
- ParseStatusUtils.toString(page.getParseStatus())).append("\n");
- sb.append("title:\t" + page.getTitle()).append("\n");
- sb.append("score:\t" + page.getScore()).append("\n");
+ ByteBuffer prevSig = page.getPrevSignature();
+ if (prevSig != null) {
+ sb.append("prevSignature:\t" + StringUtil.toHexString(prevSig.array())).append("\n");
+ }
ByteBuffer sig = page.getSignature();
if (sig != null) {
sb.append("signature:\t" + StringUtil.toHexString(sig.array())).append("\n");
}
+ sb.append("parseStatus:\t" +
+ ParseStatusUtils.toString(page.getParseStatus())).append("\n");
+ sb.append("title:\t" + page.getTitle()).append("\n");
+ sb.append("score:\t" + page.getScore()).append("\n");
+
Map<Utf8, Utf8> markers = page.getMarkers();
sb.append("markers:\t" + markers).append("\n");
-
+ sb.append("reprUrl:\t" + page.getReprUrl()).append("\n");
+ Utf8 batchId = page.getBatchId();
+ if (batchId != null) {
+ sb.append("batchId:\t" + batchId.toString()).append("\n");
+ }
Map<Utf8, ByteBuffer> metadata = page.getMetadata();
if (metadata != null) {
Iterator<Entry<Utf8, ByteBuffer>> iterator = metadata.entrySet()
Author: tejasp
Date: Fri Apr 5 23:50:56 2013
New Revision: 1465159
URL: http://svn.apache.org/r1465159
Log:
NUTCH-1031 Delegate parsing of robots.txt to crawler-commons
Added:
nutch/trunk/src/java/org/apache/nutch/protocol/RobotRulesParser.java
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
Removed:
nutch/trunk/src/java/org/apache/nutch/protocol/EmptyRobotRules.java
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/ivy/ivy.xml
nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
nutch/trunk/src/java/org/apache/nutch/protocol/Protocol.java
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
nutch/trunk/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Fri Apr 5 23:50:56 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1031 Delegate parsing of robots.txt to crawler-commons (tejasp)
+
* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
* NUTCH-1389 parsechecker and indexchecker to report truncated content (snagel)
Modified: nutch/trunk/ivy/ivy.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/ivy/ivy.xml?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/ivy/ivy.xml (original)
+++ nutch/trunk/ivy/ivy.xml Fri Apr 5 23:50:56 2013
@@ -74,6 +74,7 @@
<dependency org="oro" name="oro" rev="2.0.8" />
<dependency org="com.google.guava" name="guava" rev="11.0.2" />
+ <dependency org="com.google.code.crawler-commons" name="crawler-commons"
rev="0.2" />
<!--Configuration: test -->
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java Fri Apr 5 23:50:56 2013
@@ -51,6 +51,7 @@ import org.apache.nutch.scoring.ScoringF
import org.apache.nutch.scoring.ScoringFilters;
import org.apache.nutch.util.*;
+import crawlercommons.robots.BaseRobotRules;
/**
* A queue-based fetcher.
@@ -671,8 +672,8 @@ public class Fetcher extends Configured
}
redirecting = false;
Protocol protocol = this.protocolFactory.getProtocol(fit.url.toString());
- RobotRules rules = protocol.getRobotRules(fit.url, fit.datum);
- if (!rules.isAllowed(fit.u)) {
+ BaseRobotRules rules = protocol.getRobotRules(fit.url, fit.datum);
+ if (!rules.isAllowed(fit.u.toString())) {
// unblock
fetchQueues.finishFetchItem(fit, true);
if (LOG.isDebugEnabled()) {
Modified: nutch/trunk/src/java/org/apache/nutch/protocol/Protocol.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/protocol/Protocol.java?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/protocol/Protocol.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/protocol/Protocol.java Fri Apr 5 23:50:56 2013
@@ -25,6 +25,8 @@ import org.apache.hadoop.io.Text;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.plugin.Pluggable;
+import crawlercommons.robots.BaseRobotRules;
+
/** A retriever of url content. Implemented by protocol extensions. */
public interface Protocol extends Pluggable, Configurable {
@@ -59,5 +61,6 @@ public interface Protocol extends Plugga
* @param datum page datum
* @return robot rules (specific for this url or default), never null
*/
- RobotRules getRobotRules(Text url, CrawlDatum datum);
+ BaseRobotRules getRobotRules(Text url, CrawlDatum datum);
}
+
Added: nutch/trunk/src/java/org/apache/nutch/protocol/RobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/protocol/RobotRulesParser.java?rev=1465159&view=auto
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/protocol/RobotRulesParser.java (added)
+++ nutch/trunk/src/java/org/apache/nutch/protocol/RobotRulesParser.java Fri Apr 5 23:50:56
2013
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.protocol;
+
+// JDK imports
+import java.io.File;
+import java.io.FileReader;
+import java.io.LineNumberReader;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Hashtable;
+import java.util.StringTokenizer;
+
+// Commons Logging imports
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// Nutch imports
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.io.Files;
+
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
+import crawlercommons.robots.SimpleRobotRules.RobotRulesMode;
+import crawlercommons.robots.SimpleRobotRulesParser;
+
+/**
+ * This class uses crawler-commons for handling the parsing of {@code robots.txt} files.
+ * It emits SimpleRobotRules objects, which describe the download permissions
+ * as described in SimpleRobotRulesParser.
+ */
+public abstract class RobotRulesParser implements Configurable {
+
+ public static final Logger LOG = LoggerFactory.getLogger(RobotRulesParser.class);
+
+ protected static final Hashtable<String, BaseRobotRules> CACHE = new Hashtable<String,
BaseRobotRules> ();
+
+ /**
+ * A {@link BaseRobotRules} object appropriate for use
+ * when the {@code robots.txt} file is empty or missing;
+ * all requests are allowed.
+ */
+ public static final BaseRobotRules EMPTY_RULES = new SimpleRobotRules(RobotRulesMode.ALLOW_ALL);
+
+ /**
+ * A {@link BaseRobotRules} object appropriate for use when the
+ * {@code robots.txt} file is not fetched due to a {@code 403/Forbidden}
+ * response; all requests are disallowed.
+ */
+ public static BaseRobotRules FORBID_ALL_RULES = new SimpleRobotRules(RobotRulesMode.ALLOW_NONE);
+
+ private static SimpleRobotRulesParser robotParser = new SimpleRobotRulesParser();
+ private Configuration conf;
+ protected String agentNames;
+
+ public RobotRulesParser() { }
+
+ public RobotRulesParser(Configuration conf) {
+ setConf(conf);
+ }
+
+ /**
+ * Set the {@link Configuration} object
+ */
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+
+ // Grab the agent names we advertise to robots files.
+ String agentName = conf.get("http.agent.name");
+ if (null == agentName) {
+ throw new RuntimeException("Agent name not configured!");
+ }
+
+ String agentNames = conf.get("http.robots.agents");
+ StringTokenizer tok = new StringTokenizer(agentNames, ",");
+ ArrayList<String> agents = new ArrayList<String>();
+ while (tok.hasMoreTokens()) {
+ agents.add(tok.nextToken().trim());
+ }
+
+ /**
+ * If there are no agents for robots-parsing, use the
+ * default agent-string. If both are present, our agent-string
+ * should be the first one we advertise to robots-parsing.
+ */
+ if (agents.size() == 0) {
+ if (LOG.isErrorEnabled()) {
+ LOG.error("No agents listed in 'http.robots.agents' property!");
+ }
+ } else {
+ StringBuffer combinedAgentsString = new StringBuffer(agentName);
+ int index = 0;
+
+ if ((agents.get(0)).equalsIgnoreCase(agentName))
+ index++;
+ else if (LOG.isErrorEnabled()) {
+ LOG.error("Agent we advertise (" + agentName
+ + ") not listed first in 'http.robots.agents' property!");
+ }
+
+ // append all the agents from the http.robots.agents property
+ for(; index < agents.size(); index++) {
+ combinedAgentsString.append(", " + agents.get(index));
+ }
+
+ // always make sure "*" is included in the end
+ combinedAgentsString.append(", *");
+ this.agentNames = combinedAgentsString.toString();
+ }
+ }
+
+ /**
+ * Get the {@link Configuration} object
+ */
+ public Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Parses the robots content using the {@link SimpleRobotRulesParser} from crawler commons
+ *
+ * @param url A string containing url
+ * @param content Contents of the robots file in a byte array
+ * @param contentType The
+ * @param robotName A string containing value of
+ * @return BaseRobotRules object
+ */
+ public BaseRobotRules parseRules (String url, byte[] content, String contentType, String
robotName) {
+ return robotParser.parseContent(url, content, contentType, robotName);
+ }
+
+ public BaseRobotRules getRobotRulesSet(Protocol protocol, Text url) {
+ URL u = null;
+ try {
+ u = new URL(url.toString());
+ } catch (Exception e) {
+ return EMPTY_RULES;
+ }
+ return getRobotRulesSet(protocol, u);
+ }
+
+ public abstract BaseRobotRules getRobotRulesSet(Protocol protocol, URL url);
+
+ /** command-line main for testing */
+ public static void main(String[] argv) {
+
+ if (argv.length < 3) {
+ System.err.println("Usage: RobotRulesParser <robots-file> <url-file> <agent-names>\n");
+ System.err.println("\tThe <robots-file> will be parsed as a robots.txt file,");
+ System.err.println("\tusing the given <agent-name> to select rules. URLs ");
+ System.err.println("\twill be read (one per line) from <url-file>, and tested");
+ System.err.println("\tagainst the rules. Multiple agent names can be specified using
spaces.");
+ System.exit(-1);
+ }
+
+ try {
+ StringBuilder agentNames = new StringBuilder();
+ for(int counter = 2; counter < argv.length; counter++)
+ agentNames.append(argv[counter]).append(",");
+
+ agentNames.deleteCharAt(agentNames.length()-1);
+
+ byte[] robotsBytes = Files.toByteArray(new File(argv[0]));
+ BaseRobotRules rules = robotParser.parseContent(argv[0], robotsBytes, "text/plain",
agentNames.toString());
+
+ LineNumberReader testsIn = new LineNumberReader(new FileReader(argv[1]));
+ String testPath = testsIn.readLine().trim();
+ while (testPath != null) {
+ System.out.println( (rules.isAllowed(testPath) ? "allowed" : "not allowed") +
+ ":\t" + testPath);
+ testPath = testsIn.readLine();
+ }
+ testsIn.close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
Modified: nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
(original)
+++ nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
Fri Apr 5 23:50:56 2013
@@ -32,15 +32,16 @@ import org.apache.nutch.protocol.Protoco
import org.apache.nutch.protocol.ProtocolException;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatus;
-import org.apache.nutch.protocol.RobotRules;
import org.apache.nutch.util.GZIPUtils;
import org.apache.nutch.util.DeflateUtils;
-
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
+// crawler-commons imports
+import crawlercommons.robots.BaseRobotRules;
+
/**
* @author Jérôme Charron
*/
@@ -51,7 +52,7 @@ public abstract class HttpBase implement
private static final byte[] EMPTY_CONTENT = new byte[0];
- private RobotRulesParser robots = null;
+ private HttpRobotRulesParser robots = null;
/** The proxy hostname. */
protected String proxyHost = null;
@@ -105,7 +106,7 @@ public abstract class HttpBase implement
if (logger != null) {
this.logger = logger;
}
- robots = new RobotRulesParser();
+ robots = new HttpRobotRulesParser();
}
// Inherited Javadoc
@@ -138,7 +139,6 @@ public abstract class HttpBase implement
String urlString = url.toString();
try {
URL u = new URL(urlString);
- String host = null;
Response response = getResponse(u, datum, false); // make a request
int code = response.getCode();
@@ -381,18 +381,16 @@ public abstract class HttpBase implement
System.out.println("Content:");
String text = new String(content.getContent());
System.out.println(text);
- }
-
+ }
}
-
protected abstract Response getResponse(URL url,
CrawlDatum datum,
boolean followRedirects)
throws ProtocolException, IOException;
- public RobotRules getRobotRules(Text url, CrawlDatum datum) {
+ public BaseRobotRules getRobotRules(Text url, CrawlDatum datum) {
return robots.getRobotRulesSet(this, url);
}
-
}
+
Added: nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java?rev=1465159&view=auto
==============================================================================
--- nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
(added)
+++ nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
Fri Apr 5 23:50:56 2013
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nutch.protocol.http.api;
+
+import java.net.URL;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.net.protocols.Response;
+import org.apache.nutch.protocol.Protocol;
+import org.apache.nutch.protocol.RobotRulesParser;
+
+import crawlercommons.robots.BaseRobotRules;
+import crawlercommons.robots.SimpleRobotRules;
+
+/**
+ * This class is used for parsing robots for urls belonging to HTTP protocol.
+ * It extends the generic {@link RobotRulesParser} class and contains
+ * Http protocol specific implementation for obtaining the robots file.
+ */
+public class HttpRobotRulesParser extends RobotRulesParser {
+
+ public static final Logger LOG = LoggerFactory.getLogger(HttpRobotRulesParser.class);
+ protected boolean allowForbidden = false;
+
+ HttpRobotRulesParser() { }
+
+ public HttpRobotRulesParser(Configuration conf) {
+ super(conf);
+ allowForbidden = conf.getBoolean("http.robots.403.allow", false);
+ }
+
+ /**
+ * The hosts for which the caching of robots rules is yet to be done,
+ * it sends a Http request to the host corresponding to the {@link URL}
+ * passed, gets robots file, parses the rules and caches the rules object
+ * to avoid re-work in future.
+ *
+ * @param http The {@link Protocol} object
+ * @param url URL
+ *
+ * @return robotRules A {@link BaseRobotRules} object for the rules
+ */
+ public BaseRobotRules getRobotRulesSet(Protocol http, URL url) {
+
+ String protocol = url.getProtocol().toLowerCase(); // normalize to lower case
+ String host = url.getHost().toLowerCase(); // normalize to lower case
+
+ BaseRobotRules robotRules = (SimpleRobotRules)CACHE.get(protocol + ":" + host);
+
+ boolean cacheRule = true;
+
+ if (robotRules == null) { // cache miss
+ URL redir = null;
+ if (LOG.isTraceEnabled()) { LOG.trace("cache miss " + url); }
+ try {
+ Response response = ((HttpBase)http).getResponse(new URL(url, "/robots.txt"),
+ new CrawlDatum(), true);
+ // try one level of redirection ?
+ if (response.getCode() == 301 || response.getCode() == 302) {
+ String redirection = response.getHeader("Location");
+ if (redirection == null) {
+ // some versions of MS IIS are known to mangle this header
+ redirection = response.getHeader("location");
+ }
+ if (redirection != null) {
+ if (!redirection.startsWith("http")) {
+ // RFC says it should be absolute, but apparently it isn't
+ redir = new URL(url, redirection);
+ } else {
+ redir = new URL(redirection);
+ }
+
+ response = ((HttpBase)http).getResponse(redir, new CrawlDatum(), true);
+ }
+ }
+
+ if (response.getCode() == 200) // found rules: parse them
+ robotRules = parseRules(url.toString(), response.getContent(),
+ response.getHeader("Content-Type"),
+ agentNames);
+
+ else if ( (response.getCode() == 403) && (!allowForbidden) )
+ robotRules = FORBID_ALL_RULES; // use forbid all
+ else if (response.getCode() >= 500) {
+ cacheRule = false;
+ robotRules = EMPTY_RULES;
+ }else
+ robotRules = EMPTY_RULES; // use default rules
+ } catch (Throwable t) {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Couldn't get robots.txt for " + url + ": " + t.toString());
+ }
+ cacheRule = false;
+ robotRules = EMPTY_RULES;
+ }
+
+ if (cacheRule) {
+ CACHE.put(protocol + ":" + host, robotRules); // cache rules for host
+ if (redir != null && !redir.getHost().equals(host)) {
+ // cache also for the redirected host
+ CACHE.put(protocol + ":" + redir.getHost(), robotRules);
+ }
+ }
+ }
+ return robotRules;
+ }
+}
Modified: nutch/trunk/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
(original)
+++ nutch/trunk/src/plugin/lib-http/src/test/org/apache/nutch/protocol/http/api/TestRobotRulesParser.java
Fri Apr 5 23:50:56 2013
@@ -17,292 +17,100 @@
package org.apache.nutch.protocol.http.api;
-import org.apache.nutch.protocol.http.api.RobotRulesParser.RobotRuleSet;
-
+import crawlercommons.robots.BaseRobotRules;
import junit.framework.TestCase;
+/**
+ * JUnit test case which tests
+ * 1. that robots filtering is performed correctly as per the agent name
+ * 2. that crawl delay is extracted correctly from the robots file
+ *
+ */
public class TestRobotRulesParser extends TestCase {
- private static final String LF= "\n";
- private static final String CR= "\r";
- private static final String CRLF= "\r\n";
+
+ private static final String CONTENT_TYPE = "text/plain";
+ private static final String SINGLE_AGENT = "Agent1";
+ private static final String MULTIPLE_AGENTS = "Agent2, Agent1";
+ private static final String UNKNOWN_AGENT = "AgentABC";
+ private static final String CR = "\r";
- private static final boolean[] ACCEPT_ALL = {
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- true, // "/d",
- true, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- true, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- };
+ private static final String ROBOTS_STRING =
+ "User-Agent: Agent1 #foo" + CR
+ + "Disallow: /a" + CR
+ + "Disallow: /b/a" + CR
+ + "#Disallow: /c" + CR
+ + "Crawl-delay: 10" + CR // set crawl delay for Agent1 as 10 sec
+ + "" + CR
+ + "" + CR
+ + "User-Agent: Agent2" + CR
+ + "Disallow: /a/bloh" + CR
+ + "Disallow: /c" + CR
+ + "Disallow: /foo" + CR
+ + "Crawl-delay: 20" + CR
+ + "" + CR
+ + "User-Agent: *" + CR
+ + "Disallow: /foo/bar/" + CR; // no crawl delay for other agents
- private static final String[] ROBOTS_STRINGS= new String[] {
- "User-Agent: Agent1 #foo" + CR
- + "Disallow: /a" + CR
- + "Disallow: /b/a" + CR
- + "#Disallow: /c" + CR
- + "" + CR
- + "" + CR
- + "User-Agent: Agent2 Agent3#foo" + CR
- + "User-Agent: Agent4" + CR
- + "Disallow: /d" + CR
- + "Disallow: /e/d/" + CR
- + "" + CR
- + "User-Agent: *" + CR
- + "Disallow: /foo/bar/" + CR,
- null // Used to test EMPTY_RULES
+ private static final String[] TEST_PATHS = new String[] {
+ "http://example.com/a",
+ "http://example.com/a/bloh/foo.html",
+ "http://example.com/b",
+ "http://example.com/c",
+ "http://example.com/b/a/index.html",
+ "http://example.com/foo/bar/baz.html"
+ };
+
+ private static final boolean[] RESULTS = new boolean[] {
+ false, // /a
+ false, // /a/bloh/foo.html
+ true, // /b
+ true, // /c
+ false, // /b/a/index.html
+ true // /foo/bar/baz.html
};
- private static final String[] AGENT_STRINGS= new String[] {
- "Agent1",
- "Agent2",
- "Agent3",
- "Agent4",
- "Agent5",
- };
-
- private static final boolean[][] NOT_IN_ROBOTS_STRING= new boolean[][] {
- {
- false,
- false,
- false,
- false,
- true,
- },
- {
- false,
- false,
- false,
- false,
- true,
- }
- };
+ private HttpRobotRulesParser parser;
+ private BaseRobotRules rules;
- private static final String[] TEST_PATHS= new String[] {
- "/a",
- "/a/",
- "/a/bloh/foo.html",
- "/b",
- "/b/a",
- "/b/a/index.html",
- "/b/b/foo.html",
- "/c",
- "/c/a",
- "/c/a/index.html",
- "/c/b/foo.html",
- "/d",
- "/d/a",
- "/e/a/index.html",
- "/e/d",
- "/e/d/foo.html",
- "/e/doh.html",
- "/f/index.html",
- "/foo/bar/baz.html",
- "/f/",
- };
-
- private static final boolean[][][] ALLOWED= new boolean[][][] {
- { // ROBOTS_STRINGS[0]
- { // Agent1
- false, // "/a",
- false, // "/a/",
- false, // "/a/bloh/foo.html"
- true, // "/b",
- false, // "/b/a",
- false, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- true, // "/d",
- true, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- true, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent2
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- false, // "/d",
- false, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- false, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent3
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- false, // "/d",
- false, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- false, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent4
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- false, // "/d",
- false, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- false, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- true, // "/foo/bar.html",
- true, // "/f/",
- },
- { // Agent5/"*"
- true, // "/a",
- true, // "/a/",
- true, // "/a/bloh/foo.html"
- true, // "/b",
- true, // "/b/a",
- true, // "/b/a/index.html",
- true, // "/b/b/foo.html",
- true, // "/c",
- true, // "/c/a",
- true, // "/c/a/index.html",
- true, // "/c/b/foo.html",
- true, // "/d",
- true, // "/d/a",
- true, // "/e/a/index.html",
- true, // "/e/d",
- true, // "/e/d/foo.html",
- true, // "/e/doh.html",
- true, // "/f/index.html",
- false, // "/foo/bar.html",
- true, // "/f/",
- }
- },
- { // ROBOTS_STRINGS[1]
- ACCEPT_ALL, // Agent 1
- ACCEPT_ALL, // Agent 2
- ACCEPT_ALL, // Agent 3
- ACCEPT_ALL, // Agent 4
- ACCEPT_ALL, // Agent 5
- }
- };
-
public TestRobotRulesParser(String name) {
super(name);
+ parser = new HttpRobotRulesParser();
}
- public void testRobotsOneAgent() {
- for (int i= 0; i < ROBOTS_STRINGS.length; i++) {
- for (int j= 0; j < AGENT_STRINGS.length; j++) {
- testRobots(i, new String[] { AGENT_STRINGS[j] },
- TEST_PATHS, ALLOWED[i][j]);
- }
+ /**
+ * Test that the robots rules are interpreted correctly by the robots rules parser.
+ */
+ public void testRobotsAgent() {
+ rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(), CONTENT_TYPE,
SINGLE_AGENT);
+
+ for(int counter = 0; counter < TEST_PATHS.length; counter++) {
+ assertTrue("testing on agent (" + SINGLE_AGENT + "), and "
+ + "path " + TEST_PATHS[counter]
+ + " got " + rules.isAllowed(TEST_PATHS[counter]),
+ rules.isAllowed(TEST_PATHS[counter]) == RESULTS[counter]);
}
- }
- public void testRobotsTwoAgents() {
- for (int i= 0; i < ROBOTS_STRINGS.length; i++) {
- for (int j= 0; j < AGENT_STRINGS.length; j++) {
- for (int k= 0; k < AGENT_STRINGS.length; k++) {
- int key= j;
- if (NOT_IN_ROBOTS_STRING[i][j])
- key= k;
- testRobots(i, new String[] { AGENT_STRINGS[j], AGENT_STRINGS[k] },
- TEST_PATHS, ALLOWED[i][key]);
- }
- }
- }
- }
-
- public void testCrawlDelay() {
- RobotRulesParser p = new RobotRulesParser(new String[] { "nutchbot" });
- String delayRule1 = "User-agent: nutchbot" + CR +
- "Crawl-delay: 10" + CR +
- "User-agent: foobot" + CR +
- "Crawl-delay: 20" + CR +
- "User-agent: *" + CR +
- "Disallow:/baz" + CR;
- String delayRule2 = "User-agent: foobot" + CR +
- "Crawl-delay: 20" + CR +
- "User-agent: *" + CR +
- "Disallow:/baz" + CR;
- RobotRuleSet rules = p.parseRules(delayRule1.getBytes());
- long crawlDelay = rules.getCrawlDelay();
- assertTrue("testing crawl delay for agent nutchbot - rule 1", (crawlDelay == 10000));
- rules = p.parseRules(delayRule2.getBytes());
- crawlDelay = rules.getCrawlDelay();
- assertTrue("testing crawl delay for agent nutchbot - rule 2", (crawlDelay == -1));
- }
+ rules = parser.parseRules("testRobotsAgent", ROBOTS_STRING.getBytes(), CONTENT_TYPE,
MULTIPLE_AGENTS);
- // helper
-
- public void testRobots(int robotsString, String[] agents, String[] paths,
- boolean[] allowed) {
- String agentsString= agents[0];
- for (int i= 1; i < agents.length; i++)
- agentsString= agentsString + "," + agents[i];
- RobotRulesParser p= new RobotRulesParser(agents);
- RobotRuleSet rules= p.parseRules(ROBOTS_STRINGS[robotsString] != null
- ? ROBOTS_STRINGS[robotsString].getBytes()
- : null);
- for (int i= 0; i < paths.length; i++) {
- assertTrue("testing robots file "+robotsString+", on agents ("
- + agentsString + "), and path " + TEST_PATHS[i] + "; got "
- + rules.isAllowed(TEST_PATHS[i]) + ", rules are: " + LF
- + rules,
- rules.isAllowed(TEST_PATHS[i]) == allowed[i]);
+ for(int counter = 0; counter < TEST_PATHS.length; counter++) {
+ assertTrue("testing on agents (" + MULTIPLE_AGENTS + "), and "
+ + "path " + TEST_PATHS[counter]
+ + " got " + rules.isAllowed(TEST_PATHS[counter]),
+ rules.isAllowed(TEST_PATHS[counter]) == RESULTS[counter]);
}
}
-
-
+ /**
+ * Test that the crawl delay is extracted from the robots file for respective agent.
+ * If its not specified for a given agent, default value must be returned.
+ */
+ public void testCrawlDelay() {
+ // for SINGLE_AGENT, the crawl delay of 10 sec ie. 10000 msec must be returned by the
parser
+ rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(), CONTENT_TYPE, SINGLE_AGENT);
+ assertTrue("testing crawl delay for agent "+ SINGLE_AGENT +" : ", (rules.getCrawlDelay()
== 10000));
+
+ // for UNKNOWN_AGENT, the default crawl delay must be returned.
+ rules = parser.parseRules("testCrawlDelay", ROBOTS_STRING.getBytes(), CONTENT_TYPE, UNKNOWN_AGENT);
+ assertTrue("testing crawl delay for agent "+ UNKNOWN_AGENT +" : ", (rules.getCrawlDelay()
== Long.MIN_VALUE));
+ }
}
Modified: nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
(original)
+++ nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
Fri Apr 5 23:50:56 2013
@@ -17,35 +17,33 @@
package org.apache.nutch.protocol.file;
+import java.net.URL;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.hadoop.io.Text;
-import org.apache.nutch.metadata.Metadata;
-import org.apache.nutch.net.protocols.HttpDateFormat;
-import org.apache.nutch.net.protocols.Response;
-
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.nutch.crawl.CrawlDatum;
+import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.EmptyRobotRules;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatus;
-import org.apache.nutch.protocol.RobotRules;
+import org.apache.nutch.protocol.RobotRulesParser;
import org.apache.nutch.util.NutchConfiguration;
-import java.net.URL;
+import crawlercommons.robots.BaseRobotRules;
-/************************************
- * File.java deals with file: scheme.
- *
- * Configurable parameters are defined under "FILE properties" section
- * in ./conf/nutch-default.xml or similar.
+/**
+ * This class is a protocol plugin used for file: scheme.
+ * It creates {@link FileResponse} object and gets the content of the url from it.
+ * Configurable parameters are {@code file.content.limit} and {@code file.crawl.parent}
+ * in nutch-default.xml defined under "file properties" section.
*
* @author John Xing
- ***********************************/
+ */
public class File implements Protocol {
public static final Logger LOG = LoggerFactory.getLogger(File.class);
@@ -57,13 +55,40 @@ public class File implements Protocol {
private Configuration conf;
- // constructor
- public File() {
- }
+ public File() {}
- /** Set the point at which content is truncated. */
- public void setMaxContentLength(int length) {maxContentLength = length;}
+ /**
+ * Set the {@link Configuration} object
+ */
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ this.maxContentLength = conf.getInt("file.content.limit", 64 * 1024);
+ this.crawlParents = conf.getBoolean("file.crawl.parent", true);
+ }
+ /**
+ * Get the {@link Configuration} object
+ */
+ public Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * Set the length after at which content is truncated.
+ */
+ public void setMaxContentLength(int maxContentLength) {
+ this.maxContentLength = maxContentLength;
+ }
+
+ /**
+ * Creates a {@link FileResponse} object corresponding to the url and
+ * return a {@link ProtocolOutput} object as per the content received
+ *
+ * @param url Text containing the url
+ * @param datum The CrawlDatum object corresponding to the url
+ *
+ * @return {@link ProtocolOutput} object for the content of the file indicated by url
+ */
public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) {
String urlString = url.toString();
try {
@@ -99,11 +124,9 @@ public class File implements Protocol {
}
}
-// protected void finalize () {
-// // nothing here
-// }
-
- /** For debugging. */
+ /**
+ * Quick way for running this class. Useful for debugging.
+ */
public static void main(String[] args) throws Exception {
int maxContentLength = Integer.MIN_VALUE;
String logLevel = "info";
@@ -154,17 +177,12 @@ public class File implements Protocol {
file = null;
}
- public void setConf(Configuration conf) {
- this.conf = conf;
- this.maxContentLength = conf.getInt("file.content.limit", 64 * 1024);
- this.crawlParents = conf.getBoolean("file.crawl.parent", true);
- }
-
- public Configuration getConf() {
- return this.conf;
- }
-
- public RobotRules getRobotRules(Text url, CrawlDatum datum) {
- return EmptyRobotRules.RULES;
+ /**
+ * No robots parsing is done for file protocol.
+ * So this returns a set of empty rules which will allow every url.
+ */
+ public BaseRobotRules getRobotRules(Text url, CrawlDatum datum) {
+ return RobotRulesParser.EMPTY_RULES;
}
}
+
Modified: nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java?rev=1465159&r1=1465158&r2=1465159&view=diff
==============================================================================
--- nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java (original)
+++ nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java Fri
Apr 5 23:50:56 2013
@@ -24,30 +24,33 @@ import org.apache.commons.net.ftp.FTPFil
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.hadoop.io.Text;
-import org.apache.nutch.net.protocols.HttpDateFormat;
import org.apache.nutch.net.protocols.Response;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.protocol.Content;
-import org.apache.nutch.protocol.EmptyRobotRules;
+import org.apache.nutch.protocol.RobotRulesParser;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatus;
-import org.apache.nutch.protocol.RobotRules;
+
+import crawlercommons.robots.BaseRobotRules;
import java.net.URL;
import java.io.IOException;
-/************************************
- * Ftp.java deals with ftp: scheme.
- *
- * Configurable parameters are defined under "FTP properties" section
- * in ./conf/nutch-default.xml or similar.
+/**
+ * This class is a protocol plugin used for ftp: scheme.
+ * It creates {@link FtpResponse} object and gets the content of the url from it.
+ * Configurable parameters are {@code ftp.username}, {@code ftp.password},
+ * {@code ftp.content.limit}, {@code ftp.timeout},
+ * {@code ftp.server.timeout}, {@code ftp.password},
+ * {@code ftp.keep.connection} and {@code ftp.follow.talk}.
+ * For details see "FTP properties" section in {@code nutch-default.xml}.
*
* @author John Xing
- ***********************************/
+ */
public class Ftp implements Protocol {
public static final Logger LOG = LoggerFactory.getLogger(Ftp.class);
@@ -106,6 +109,15 @@ public class Ftp implements Protocol {
this.keepConnection = keepConnection;
}
+ /**
+ * Creates a {@link FtpResponse} object corresponding to the url and
+ * returns a {@link ProtocolOutput} object as per the content received
+ *
+ * @param url Text containing the ftp url
+ * @param datum The CrawlDatum object corresponding to the url
+ *
+ * @return {@link ProtocolOutput} object for the url
+ */
public ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum) {
String urlString = url.toString();
try {
@@ -216,7 +228,9 @@ public class Ftp implements Protocol {
ftp = null;
}
-
+ /**
+ * Set the {@link Configuration} object
+ */
public void setConf(Configuration conf) {
this.conf = conf;
this.maxContentLength = conf.getInt("ftp.content.limit", 64 * 1024);
@@ -228,12 +242,20 @@ public class Ftp implements Protocol {
this.followTalk = conf.getBoolean("ftp.follow.talk", false);
}
+ /**
+ * Get the {@link Configuration} object
+ */
public Configuration getConf() {
return this.conf;
}
- public RobotRules getRobotRules(Text url, CrawlDatum datum) {
- return EmptyRobotRules.RULES;
+ /**
+ * Currently, no robots parsing is done for ftp protocol
+ * and this returns a set of empty rules which will allow every url.
+ * There a jira logged for the same NUTCH-1513
+ */
+ public BaseRobotRules getRobotRules(Text url, CrawlDatum datum) {
+ return RobotRulesParser.EMPTY_RULES;
}
-
}
+
Author: lewismc
Date: Wed Apr 3 21:15:07 2013
New Revision: 1464207
URL: http://svn.apache.org/r1464207
Log:
NUTCH-1552 possibility of a NPE in index-more plugin
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1464207&r1=1464206&r2=1464207&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Wed Apr 3 21:15:07 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1552 possibility of a NPE in index-more plugin (kaveh minooie via lewismc)
+
* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
* NUTCH-1389 parsechecker and indexchecker to report truncated content (snagel)
Modified: nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=1464207&r1=1464206&r2=1464207&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
(original)
+++ nutch/branches/2.x/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
Wed Apr 3 21:15:07 2013
@@ -199,7 +199,7 @@ public class MoreIndexingFilter implemen
doc.add("type", mimeType);
// Check if we need to split the content type in sub parts
- if (conf.getBoolean("moreIndexingFilter.indexMimeTypeParts", true)) {
+ if ( null != contentType && conf.getBoolean("moreIndexingFilter.indexMimeTypeParts",
true)) {
String[] parts = getParts(contentType.toString());
for(String part: parts) {
Author: fenglu
Date: Thu Mar 28 13:09:09 2013
New Revision: 1462079
URL: http://svn.apache.org/r1462079
Log:
NUTCH-1547 BasicIndexingFilter - Problem to index full title
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/conf/nutch-default.xml
nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1462079&r1=1462078&r2=1462079&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Thu Mar 28 13:09:09 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
+
* NUTCH-1389 parsechecker and indexchecker to report truncated content (snagel)
* NUTCH-1419 parsechecker and indexchecker to report protocol status (snagel via lewismc)
Modified: nutch/branches/2.x/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/conf/nutch-default.xml?rev=1462079&r1=1462078&r2=1462079&view=diff
==============================================================================
--- nutch/branches/2.x/conf/nutch-default.xml (original)
+++ nutch/branches/2.x/conf/nutch-default.xml Thu Mar 28 13:09:09 2013
@@ -752,7 +752,7 @@
<property>
<name>indexer.max.title.length</name>
<value>100</value>
- <description>The maximum number of characters of a title that are indexed.
+ <description>The maximum number of characters of a title that are indexed. A value
of -1 disables this check.
Used by index-basic.
</description>
</property>
Modified: nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java?rev=1462079&r1=1462078&r2=1462079&view=diff
==============================================================================
--- nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
(original)
+++ nutch/branches/2.x/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
Thu Mar 28 13:09:09 2013
@@ -109,7 +109,7 @@ public class BasicIndexingFilter impleme
// title
String title = TableUtil.toString(page.getTitle());
- if (title.length() > MAX_TITLE_LENGTH) { // truncate title if needed
+ if (MAX_TITLE_LENGTH > -1 && title.length() > MAX_TITLE_LENGTH) { // truncate
title if needed
title = title.substring(0, MAX_TITLE_LENGTH);
}
if (title.length() > 0) {
Author: fenglu
Date: Thu Mar 28 13:04:28 2013
New Revision: 1462078
URL: http://svn.apache.org/r1462078
Log:
NUTCH-1547 BasicIndexingFilter - Problem to index full title
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1462078&r1=1462077&r2=1462078&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Thu Mar 28 13:04:28 2013
@@ -2,6 +2,8 @@ Nutch Change Log
(trunk): Current Development
+* NUTCH-1547 BasicIndexingFilter - Problem to index full title (Feng)
+
* NUTCH-1389 parsechecker and indexchecker to report truncated content (snagel)
* NUTCH-1419 parsechecker and indexchecker to report protocol status (snagel + lewismc)
Modified: nutch/trunk/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1462078&r1=1462077&r2=1462078&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Thu Mar 28 13:04:28 2013
@@ -897,7 +897,7 @@
<property>
<name>indexer.max.title.length</name>
<value>100</value>
- <description>The maximum number of characters of a title that are indexed.
+ <description>The maximum number of characters of a title that are indexed. A value
of -1 disables this check.
</description>
</property>
Modified: nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java?rev=1462078&r1=1462077&r2=1462078&view=diff
==============================================================================
--- nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
(original)
+++ nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
Thu Mar 28 13:04:28 2013
@@ -108,7 +108,7 @@ public class BasicIndexingFilter impleme
// title
String title = parse.getData().getTitle();
- if (title.length() > MAX_TITLE_LENGTH) { // truncate title if needed
+ if (MAX_TITLE_LENGTH > -1 && title.length() > MAX_TITLE_LENGTH) {
// truncate title if needed
title = title.substring(0, MAX_TITLE_LENGTH);
}