lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmarsch...@apache.org
Subject cvs commit: jakarta-lucene-sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher FetcherTaskQueue.java
Date Tue, 22 Oct 2002 15:09:22 GMT
cmarschner    2002/10/22 08:09:22

  Modified:    contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher
                        FetcherTaskQueue.java
  Log:
  removed test case from main()
  
  Revision  Changes    Path
  1.6       +116 -73   jakarta-lucene-sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherTaskQueue.java
  
  Index: FetcherTaskQueue.java
  ===================================================================
  RCS file: /home/cvs/jakarta-lucene-sandbox/contributions/webcrawler-LARM/src/de/lanlab/larm/fetcher/FetcherTaskQueue.java,v
  retrieving revision 1.5
  retrieving revision 1.6
  diff -u -r1.5 -r1.6
  --- FetcherTaskQueue.java	15 Sep 2002 20:20:35 -0000	1.5
  +++ FetcherTaskQueue.java	22 Oct 2002 15:09:22 -0000	1.6
  @@ -59,6 +59,7 @@
   import de.lanlab.larm.util.CachingQueue;
   import de.lanlab.larm.util.HashedCircularLinkedList;
   import java.net.URL;
  +import de.lanlab.larm.net.*;
   
   /**
    * this special kind of task queue reorders the incoming tasks so that every subsequent
  @@ -84,7 +85,10 @@
       /**
        * Constructor for the FetcherTaskQueue object. Does nothing
        */
  -    public FetcherTaskQueue() { }
  +    public FetcherTaskQueue(HostManager manager)
  +    {
  +        this.manager = manager;
  +    }
   
   
       /**
  @@ -156,6 +160,7 @@
           return servers.size();
       }
   
  +    HostManager manager;
       /**
        * get the next task. warning: not synchronized
        *
  @@ -164,17 +169,53 @@
       public Object remove()
       {
           FetcherTask t = null;
  +        String start=null;
           if (servers.size() > 0)
           {
  -            Queue q = (Queue) servers.next();
  -            // assert(q != null && q.size() > 0)
  -            t = (FetcherTask)q.remove();
  -            if (q.size() == 0)
  -            {
  -                servers.removeCurrent();
  -                q = null;
  -            }
  -            size--;
  +//            while(true)
  +//            {
  +                Queue q = (Queue) servers.next();
  +                String host = (String)servers.getCurrentKey();
  +//                if(start == null)
  +//                {
  +//                    start = host;
  +//                }
  +//                else if(host.equals(start))
  +//                {
  +//                    System.out.println("FetcherTaskQueue: all hosts busy. waiting 1sec");
  +//                    try
  +//                    {
  +//                        Thread.sleep(1000);
  +//                    }
  +//                    catch(InterruptedException e)
  +//                    {
  +//                        break;
  +//                    }
  +//                }
  +//                HostInfo hInfo = manager.getHostInfo(host);
  +//                System.out.println("getting sync on " + hInfo.getHostName());
  +//                synchronized(hInfo.getLockMonitor())
  +//                {
  +//                    if(!hInfo.isBusy())
  +//                    {
  +//                        System.out.println("FetcherTaskQueue: host " + host + " ok");
  +//                        hInfo.obtainLock(); // decreased in FetcherTask
  +                        // assert(q != null && q.size() > 0)
  +                        t = (FetcherTask)q.remove();
  +                        if (q.size() == 0)
  +                        {
  +                            servers.removeCurrent();
  +                            q = null;
  +                        }
  +                        size--;
  +//                        break;
  +//                    }
  +//                    else
  +//                    {
  +//                        System.out.println("FetcherTaskQueue: host " + host + " is busy.
next...");
  +//                    }
  +//                }
  +//            }
           }
           return t;
       }
  @@ -187,68 +228,70 @@
        */
       public static void main(String args[])
       {
  -        FetcherTaskQueue q = new FetcherTaskQueue();
  -        de.lanlab.larm.net.HostManager hm = new de.lanlab.larm.net.HostManager(10);
  -        System.out.println("Test 1. put in 4 yahoos and 3 lmus. pull out LMU/Yahoo/LMU/Yahoo/LMU/Yahoo/Yahoo");
  -        try
  -        {
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null,
false, null, hm)));
  -        }
  -        catch (Throwable t)
  -        {
  -            t.printStackTrace();
  -        }
  +// FIXME: put that into a JUnit test case
  +//        FetcherTaskQueue q = new FetcherTaskQueue();
  +//        de.lanlab.larm.net.HostResolver hm = new de.lanlab.larm.net.HostResolver();
  +//        System.out.println("Test 1. put in 4 yahoos and 3 lmus. pull out LMU/Yahoo/LMU/Yahoo/LMU/Yahoo/Yahoo");
  +//        try
  +//        {
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null,
URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null,
URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null,
URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//        }
  +//        catch (Throwable t)
  +//        {
  +//            t.printStackTrace();
  +//        }
  +//
  +//        System.out.println(((FetcherTask) q.remove()).getInfo());
  +//        System.out.println(((FetcherTask) q.remove()).getInfo());
  +//        System.out.println(((FetcherTask) q.remove()).getInfo());
  +//        System.out.println(((FetcherTask) q.remove()).getInfo());
  +//        System.out.println(((FetcherTask) q.remove()).getInfo());
  +//        System.out.println(((FetcherTask) q.remove()).getInfo());
  +//        System.out.println(((FetcherTask) q.remove()).getInfo());
  +//
  +//        System.out.println("Test 2. new Queue");
  +//        q = new FetcherTaskQueue();
  +//        System.out.println("size [0]:");
  +//        System.out.println(q.size());
  +//        try
  +//        {
  +//            System.out.println("put 3 lmus.");
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null,
URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null,
URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null,
URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            System.out.print("pull out 1st element [lmu/1]: ");
  +//            System.out.println(((FetcherTask) q.remove()).getInfo());
  +//            System.out.println("size now [2]: " + q.size());
  +//            System.out.print("pull out 2nd element [lmu/2]: ");
  +//            System.out.println(((FetcherTask) q.remove()).getInfo());
  +//            System.out.println("size now [1]: " + q.size());
  +//            System.out.println("put in 3 yahoos");
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  +//            System.out.println("Size now [3]: " + q.size());
  +//            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  +//            System.out.println("Size now [2]: " + q.size());
  +//            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  +//            System.out.println("Size now [1]: " + q.size());
  +//            System.out.println("put in another Yahoo");
  +//            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"),
null, URLMessage.LINKTYPE_ANCHOR, null, hm)));
  +//            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  +//            System.out.println("Size now [1]: " + q.size());
  +//            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  +//            System.out.println("Size now [0]: " + q.size());
  +//        }
  +//        catch (Throwable t)
  +//        {
  +//            t.printStackTrace();
  +//        }
   
  -        System.out.println(((FetcherTask) q.remove()).getInfo());
  -        System.out.println(((FetcherTask) q.remove()).getInfo());
  -        System.out.println(((FetcherTask) q.remove()).getInfo());
  -        System.out.println(((FetcherTask) q.remove()).getInfo());
  -        System.out.println(((FetcherTask) q.remove()).getInfo());
  -        System.out.println(((FetcherTask) q.remove()).getInfo());
  -        System.out.println(((FetcherTask) q.remove()).getInfo());
  -
  -        System.out.println("Test 2. new Queue");
  -        q = new FetcherTaskQueue();
  -        System.out.println("size [0]:");
  -        System.out.println(q.size());
  -        try
  -        {
  -            System.out.println("put 3 lmus.");
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/1"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/2"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.lmu.de/3"), null,
false, null, hm)));
  -            System.out.print("pull out 1st element [lmu/1]: ");
  -            System.out.println(((FetcherTask) q.remove()).getInfo());
  -            System.out.println("size now [2]: " + q.size());
  -            System.out.print("pull out 2nd element [lmu/2]: ");
  -            System.out.println(((FetcherTask) q.remove()).getInfo());
  -            System.out.println("size now [1]: " + q.size());
  -            System.out.println("put in 3 yahoos");
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/1"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/2"), null,
false, null, hm)));
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/3"), null,
false, null, hm)));
  -            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  -            System.out.println("Size now [3]: " + q.size());
  -            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  -            System.out.println("Size now [2]: " + q.size());
  -            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  -            System.out.println("Size now [1]: " + q.size());
  -            System.out.println("put in another Yahoo");
  -            q.insert(new FetcherTask(new URLMessage(new URL("http://www.yahoo.de/4"), null,
false, null, hm)));
  -            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  -            System.out.println("Size now [1]: " + q.size());
  -            System.out.println("remove [?]: " + ((FetcherTask) q.remove()).getInfo());
  -            System.out.println("Size now [0]: " + q.size());
  -        }
  -        catch (Throwable t)
  -        {
  -            t.printStackTrace();
  -        }
       }
   }
  
  
  

--
To unsubscribe, e-mail:   <mailto:lucene-dev-unsubscribe@jakarta.apache.org>
For additional commands, e-mail: <mailto:lucene-dev-help@jakarta.apache.org>


Mime
View raw message