perl-docs-cvs mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mose...@apache.org
Subject cvs commit: modperl-docs/src/search spider.pl
Date Sat, 20 Apr 2002 06:13:05 GMT
moseley     02/04/19 23:13:05

  Modified:    src/search spider.pl
  Log:
  Add debugging and quiet mode for the spider.
  
  perldoc spider.pl
  
  Revision  Changes    Path
  1.4       +47 -7     modperl-docs/src/search/spider.pl
  
  Index: spider.pl
  ===================================================================
  RCS file: /home/cvs/modperl-docs/src/search/spider.pl,v
  retrieving revision 1.3
  retrieving revision 1.4
  diff -u -r1.3 -r1.4
  --- spider.pl	3 Mar 2002 11:27:22 -0000	1.3
  +++ spider.pl	20 Apr 2002 06:13:05 -0000	1.4
  @@ -2,7 +2,7 @@
   use strict;
   
   
  -# $Id: spider.pl,v 1.3 2002/03/03 11:27:22 stas Exp $
  +# $Id: spider.pl,v 1.4 2002/04/20 06:13:05 moseley Exp $
   #
   # "prog" document source for spidering web servers
   #
  @@ -23,7 +23,7 @@
   use HTML::Tagset;
   
   use vars '$VERSION';
  -$VERSION = sprintf '%d.%02d', q$Revision: 1.3 $ =~ /: (\d+)\.(\d+)/;
  +$VERSION = sprintf '%d.%02d', q$Revision: 1.4 $ =~ /: (\d+)\.(\d+)/;
   
   use vars '$bit';
   use constant DEBUG_ERRORS   => $bit = 1;    # program errors
  @@ -34,6 +34,17 @@
   use constant DEBUG_INFO     => $bit <<= 1;  # more verbose
   use constant DEBUG_LINKS    => $bit <<= 1;  # prints links as they are extracted
   
  +my %DEBUG_MAP = (
  +    errors  => DEBUG_ERRORS,
  +    url     => DEBUG_URL,
  +    headers => DEBUG_HEADERS,
  +    failed  => DEBUG_FAILED,
  +    skipped => DEBUG_SKIPPED,
  +    info    => DEBUG_INFO,
  +    links   => DEBUG_LINKS,
  +);
  +    
  +
   
   use constant MAX_SIZE       => 5_000_000;   # Max size of document to fetch
   use constant MAX_WAIT_TIME  => 30;          # request time.
  @@ -62,7 +73,7 @@
       }
   
   
  -    print STDERR "$0: Reading parameters from '$config'\n";
  +    print STDERR "$0: Reading parameters from '$config'\n" unless $ENV{SPIDER_QUIET};
   
       my $abort;
       local $SIG{HUP} = sub { warn "Caught SIGHUP\n"; $abort++ } unless $^O =~ /Win32/i;
  @@ -103,8 +114,18 @@
   
       # set defaults
   
  -    $server->{debug} ||= 0;
  -    die "debug parameter '$server->{debug}' must be a number\n" unless $server->{debug}
=~ /^\d+$/;
  +    if ( $ENV{SPIDER_DEBUG} ) {
  +        $server->{debug} = 0;
  +
  +        $server->{debug} |= (exists $DEBUG_MAP{lc $_} ? $DEBUG_MAP{lc $_} : die "Bad
debug setting passed in environment '$_'\nOptions are: " . join( ', ', keys %DEBUG_MAP) ."\n")
  +            for split /\s*,\s*/, $ENV{SPIDER_DEBUG};
  +
  +    } else {
  +        $server->{debug} ||= 0;
  +        die "debug parameter '$server->{debug}' must be a number\n" unless $server->{debug}
=~ /^\d+$/;
  +    }
  +
  +    $server->{quiet} ||= $ENV{SPIDER_QUIET} || 0;
   
   
       $server->{max_size} ||= MAX_SIZE;
  @@ -137,7 +158,7 @@
       my $start = time;
   
       if ( $server->{skip} ) {
  -        print STDERR "Skipping: $server->{base_url}\n";
  +        print STDERR "Skipping: $server->{base_url}\n" unless $server->{quiet};
           return;
       }
   
  @@ -235,6 +256,9 @@
       eval { spider( $server, $uri ) };
       print STDERR $@ if $@;
   
  +    return if $server->{quiet};
  +
  +
       $start = time - $start;
       $start++ unless $start;
   
  @@ -246,6 +270,7 @@
           $max_num = length $val if length $val > $max_num;
       }
   
  +
       printf STDERR "\nSummary for: $server->{base_url}\n";
   
       for ( sort keys %{$server->{counts}} ) {
  @@ -468,7 +493,7 @@
           return;
       }
   
  -    $response->request->uri->userinfo( undef );
  +    $response->request->uri->userinfo( undef ) if $response->request;
   
   
       # skip excluded by robots.txt
  @@ -1339,6 +1364,21 @@
   
   And you will see debugging info as it runs, and the fetched documents will be saved
   in the C<spider.out> file.
  +
  +Debugging can be also be set by an environment variable when running swish.  This will
  +override any setting in the configuration file.  Set the variable SPIDER_DEBUG when running
  +the spider.  You can specify any of the above debugging options, separated by a comma.
  +
  +For example with Bourne type shell:
  +
  +    SPIDER_DEBUG=url,links
  +
  +=item quiet
  +
  +If this is true then normal, non-error messages will be supressed.  Quiet mode can also
  +be set by setting the environment variable SPIDER_QUIET to any true value.
  +
  +    SPIDER_QUIET=1
   
   =item max_depth
   
  
  
  

---------------------------------------------------------------------
To unsubscribe, e-mail: docs-cvs-unsubscribe@perl.apache.org
For additional commands, e-mail: docs-cvs-help@perl.apache.org


Mime
View raw message