Mailing List Archive

cmarschner 2002/10/22 08:11:44

Modified: contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher
FetcherThread.java
Log:
moved HTTPClient config from Task to Thread

Revision Changes Path
1.4 +20 -1 jakarta-lucene-sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherThread.java

Index: FetcherThread.java
===================================================================
RCS file: /home/cvs/jakarta-lucene-sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherThread.java,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- FetcherThread.java 17 Jun 2002 13:59:28 -0000 1.3
+++ FetcherThread.java 22 Oct 2002 15:11:44 -0000 1.4
@@ -57,6 +57,7 @@
import de.lanlab.larm.threads.ServerThread;
import de.lanlab.larm.util.State;
import de.lanlab.larm.net.HostManager;
+import HTTPClient.NVPair;

/**
* a server thread for the thread pool that records the number
@@ -75,6 +76,22 @@

byte[] documentBuffer = new byte[Constants.FETCHERTASK_READSIZE];

+ /**
+ * default headers for HTTPClient
+ */
+ private volatile NVPair headers[] = new NVPair[2];
+
+
+ public NVPair[] getDefaultHeaders()
+ {
+ return headers;
+ }
+
+ public int getUsedDefaultHeaders()
+ {
+ return 1;
+ }
+
public HostManager getHostManager()
{
return hostManager;
@@ -84,6 +101,8 @@
{
super(threadNumber,"FetcherThread " + threadNumber, threadGroup);
this.hostManager = hostManager;
+ headers[0] = new HTTPClient.NVPair("User-Agent", Constants.CRAWLER_AGENT);
+ headers[1] = null; // may contain an additional field
}

public static String STATE_IDLE = "Idle";




--
To unsubscribe, e-mail: <mailto:lucene-dev-unsubscribe@jakarta.apache.org>
For additional commands, e-mail: <mailto:lucene-dev-help@jakarta.apache.org>