Thanks for your help.
Joe
I have included a copy of the file as requested (see below)
*****START*****
#!/usr/local/bin/perl -w
use CGI::Carp qw/fatalsToBrowser/;
require "admin/links.cfg"; # Change this to full path to links.cfg if you have problems.
require "$db_lib_path/db_utils.pl";
require "$db_lib_path/links.def";
use Socket;
$Rules{'Hits Per Page'} = 10;
$Rules{'Multiplier: URL'} = 4;
$Rules{'Multiplier: Title'} = 10;
$Rules{'Multiplier: Keyword'} = 10;
$Rules{'Multiplier: Description'} = 4;
$Rules{'Max Characters: URL'} = 128;
$Rules{'Max Characters: Title'} = 96;
$Rules{'Max Characters: Description'} = 384;
$Rules{'Max Characters: Auto Description'} = 150;
$Rules{'Max Characters: Keywords'} = 256;
$Rules{'Max Characters: File'} = 64000;
$Rules{'Forbid All Cap Titles'} = 1;
$Rules{'Forbid All Cap Descriptions'} = 1;
$Rules{'Crawler: Minimum WhiteSpace'} = 0.01;
$Rules{'Crawler: Max Pages Per Batch'} = 12;
$Rules{'Crawler: Max Redirects'} = 6;
$Rules{'Crawler: Days Til Refresh'} = 30;
$Rules{'Crawler: User Agent'} = 'Mozilla/4.0 (compatible: FDSE robot)';
$Rules{'Crawler: Follow Query Strings'} = 0;
$Rules{'Crawler: Rogue'} = 0;
@PromoteSites = "";
$Rules{'Promote Value'} = 20;
@IgnoredWords = ('a','about','all','an','and','any','are','as','at',
'be','been','by','can','do','find','for','from','get','have','he',
'how','htm','html','http','i','if','in','is','it','me','most','new',
'no','not','of','on','one','or','other','page','s','site',
'that','the','this','to','two','use','w','web','what','when','where',
'which','who','why','will','with','you','your');
open (ID, "<$db_spider_id_file_name") or &error("error in process_form. unable to open id file: $db_spider_id_file_name. Reason: $!");
$link_hits = <ID>; # Get next ID number
close ID;
$link_blocks = 0;
open(FILE, $db_spidered_name) or die "Can't open `$filename': $!";
while (sysread FILE, $buffer, 4096) {
$link_blocks += ($buffer =~ tr/\n//);
}
close FILE;
@MonthNames = ('Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec');
$stuff = -s $db_spider_name;
$size = &filesize($stuff);
my %FORM = &ReadInput;
$|=1;
print "Content-type: text/html\n\n";
print qq~
<html>
<head>
<title>$build_site_title goFetch Spider</title>
<meta name="Robots" content="noindex">
<meta name="Robots" content="nofollow">
</head>
<body bgcolor="#FFFFFF" text="#000080" link="#3399FF" vlink="#3399FF" alink="#0000FF">
//isg;
$mydescrip =~ s/
//isg;
$mydescrip =~ s/<p>//isg;
$mydescrip =~ s,\n,,g;
$mydescrip = substr($mydescrip,0,225);
if (length($mydescrip) > 224) {
$mydescrip =~ s/\s+\S*$/.../;
}
}
$Month = $MonthNames[$MM];
$mykeywords = $res->header('X-Meta-Keywords');
$mykeywords =~ s/
//isg;
$mykeywords =~ s/<p>//isg;
$mykeywords =~ s,\n,,g;
$mykeywords = substr($mykeywords,0,145);
if (length($mykeywords) > 144) {
$mykeywords =~ s/\s+\S*$/.../;
}
use HTTP::Date;
$lastupd = time2str($res->last_modified);
use LWP::Simple;
$stuff = get($url);
$myurl = $_;
$mysize = &filesize(length($stuff));
$endsub = time() - $start;
print "$LinkCount <a href=\"$_$QueryString\"><b>$mytitle</b></a> - $mydescrip
Keywords: $mykeywords
<i><a href=\"$_$QueryString\">$_</a><small> - size $mysize - last updated $lastupd</i></small><p>\n";
$LinkCount++;
# Update the counter.
open (ID, "<$db_spider_id_file_name") or &cgierr("error in process_form. unable to open id file: $db_spider_id_file_name. Reason: $!");
$ID = <ID> + 1; # Get next ID number
close ID;
# Update the counter.
open (ID, ">$db_spider_id_file_name") or &cgierr("error in get_defaults. unable to open id file: $db_links_id_file_name. Reason: $!");
flock(ID, 2) unless (!$db_use_flock);
print ID $ID; # update counter.
close ID; # automatically removes file lock
open (SPIDER, ">>$db_spider_name") or &cgierr ("Can't open for output counter file. Reason: $!");
if ($db_use_flock) { flock (SPIDER, 2) or &cgierr ("Can't get file lock. Reason: $!"); }
print SPIDER "$ID%%$mytitle%%$myurl%%$mydescrip%%$mykeywords%%$mysize%%$lastupd\n";
close SPIDER;
}
}
else {
print <<"EOM";
No embedded links were found during this crawl session
EOM
}
print <<"EOM";
</body>
$LinkCount links fetched during this crawl session
</html>
EOM
open (LC, ">$db_linkcount") or &cgierr("error in get_defaults. unable to open id file: $db_links_id_file_name. Reason: $!");
flock(LC, 2) unless (!$db_use_flock);
print LC $LinkCount; # update counter.
close LC; # automatically removes file lock
}
sub LinkCount {
$LinkCount;
}
sub filesize {
my $bytes = shift;
my @info = qw!bytes kb mb gb tb!;
my ($i);
while ($bytes > 1024) {
$bytes = int($bytes/1024) + 1;
$i++;
}
return "$bytes $info[$i]";
}
sub CompileLinks {
foreach (@SavedLinks) {
$SaveLinks{$_} = 1;
# push(@Global,"$_");
}
}
sub MakeRecord {
my ($URL, $LastModT, $sText) = @_;
$FBYTES = sprintf('.f',length($sText));
($Title,$Description,$sText,$Links) = &Extract_Meta($sText,$URL);
$AlphaData = ' ';
$AlphaData .= "u= $URL ";
$AlphaData .= "t= $Title ";
$AlphaData .= "d= $Description ";
$AlphaData .= 'uM='.CompressStrip($URL);
$AlphaData .= 'h='.CompressStrip($sText);
$AlphaData .= 'l='.CompressStrip($Links);
$LastModT = $LastModT ? $LastModT : time;
($DD,$MM,$YYYY) = (localtime($LastModT))[3..5];
$YYYY += 1900;
$CC = 1;
foreach (@PromoteSites) {
next unless ($URL =~ m!^$_!i);
$CC = $Rules{'Promote Value'};
last;
}
for ($CC,$DD,$MM) {
$_ = sprintf('.f',$_);
}
return "$CC$DD$MM$YYYY$FBYTES$AlphaData\n";
}
sub Extract_Meta {
($HTML_Text,$URL) = @_;
($Title, $Description, $Links) = ('','','');
foreach (split(m!\<A !i, $HTML_Text)) {
next unless (m!^([^\>]*)HREF(\s+)?=(\s+)?\"?([^\"\s\>]+)!i);
$ThisLink = $4;
$Links .= ' '.$ThisLink;
next unless ($SetSaveLinks == 1);
next if (($Rules{'Crawler: Follow Query Strings'} == 0) && ($ThisLink =~ m!\?!));
$ThisLink = &GetAbsoluteAddress($ThisLink,$URL);
push(@SavedLinks,$ThisLink) if ($ThisLink);
}
foreach (split(m!\<I?FRAME !i, $HTML_Text)) {
next unless (m!^([^\>]*)SRC(\s+)?=(\s+)?\"?([^\"\s\>]+)!i);
$ThisLink = $4;
$Links .= ' '.$ThisLink;
next unless ($SetSaveLinks == 1);
next if (($Rules{'Crawler: Follow Query Strings'} == 0) && ($ThisLink =~ m!\?!));
$ThisLink = &GetAbsoluteAddress($ThisLink,$URL);
push(@SavedLinks,$ThisLink) if ($ThisLink);
}
$HTML_Text .= ' || ';
if ($HTML_Text =~ m!<TITLE.*?>(.*?)<!i) {
$Title = ' '.$1;
$HTML_Text =~ s!<TITLE.*?>.*?<\/TITLE>! !i;
$HTML_Text .= $Title x $Rules{'Multiplier: Title'};
}
elsif (($FILE) && ($FILE =~ m!([^\/]+)$!)) {
$Title = $1;
}
elsif ($URL =~ m!([^\/]+)$!) {
$Title = $1;
}
elsif ($FILE) {
$Title = $FILE;
}
elsif ($URL) {
$Title = $URL;
}
else {
$Title = 'Document';
}
if (($Rules{'Forbid All Cap Titles'}) && ($Title !~ m![a-z]!)) {
$NewTitle = '';
foreach (split(m!\s+!,$Title)) {
unless (length($_) > 1) {
$NewTitle .= $_.' ';
next
}
$NewTitle .= ' '.substr($_,0,1);
$_ = substr($_,1,(length($_)-1));
tr[A-Z][a-z];
$NewTitle .= $_;
}
$Title = $NewTitle;
}
if ($HTML_Text =~ m!.*?<META([^\>]*?)(NAME|HTTP-EQUIV)="keywords"([^\>]*?)(CONTENT|VALUE)="([^\"]+)"!i) {
$KeyWords = ' '.$5;
$HTML_Text .= $KeyWords x $Rules{'Multiplier: Keyword'};
}
if ($HTML_Text =~ m!.*?<META([^\>]*?)(NAME|HTTP-EQUIV)="description"([^\>]*?)(CONTENT|VALUE)="([^\"]+)"!i) {
$Description = ' '.$5;
$HTML_Text .= $Description x $Rules{'Multiplier: Description'};
}
$HTML_Text =~ s/<[^>]*\s+ALT\s*=\s*"(([^>"])*)"[^>]*>/ $1 /ig;
$NoScript = '';
foreach (split(m!(\<\/SCRIPT>|\<\/STYLE>)!i, $HTML_Text)) {
next unless $_;
if (m!^(.*)(\<SCRIPT|\<STYLE)!i) {
$NoScript .= ' '.$1;
}
else {
$NoScript .= ' '.$_;
}
}
$HTML_Text = $NoScript;
if ($HTML_Text =~ m!(.*)<NOFRAMES>(.*)</NOFRAMES>(.*?)!i) {
if (length($2) < 2000) {
$HTML_Text = $1.' '.$2;
}
}
$HTML_Text =~ s!<([^>]*?)>! !g;
$HTML_Text =~ s!\s+! !g;
unless ($Description) {
$tempDescription = substr($HTML_Text,0,$Rules{'Max Characters: Auto Description'});
if ($tempDescription =~ m!([^\|]*)\s+!) {
$Description = $1.'...';
}
else {
$Description = 'No description available.';
}
}
$HTML_Text =~ s!(\W|\_)! !g;
$Title =~ s!\s+! !g;
if ($Title =~ m!^ (.+)!) {
$Title = $1;
}
$Description =~ s!\s+! !g;
if ($Description =~ m!^ (.+)!) {
$Description = $1;
}
if (($Rules{'Forbid All Cap Descriptions'}) && ($Description !~ /[a-z]/)) {
$NewDescription = '';
foreach (split(/\s+/,$Description)) {
$NewDescription .= ' '.substr($_,0,1);
$_ = substr($_,1,(length($_)-1));
tr[A-Z][a-z];
$NewDescription .= $_;
}
$Description = $NewDescription;
}
return($Title,$Description,$HTML_Text,$Links);
}
sub AdminVersion {
my ($Rank,$URL,$Title,$Description,$Size,$Day,$Month,$Year) = @_;
$Size = ($Size<1500)?int($Size).' bytes':(int($Size/1000)).' Kilobytes';
$wURL = webEncode($URL);
$Description =~ s/
//isg;
$Description =~ s,\n,,g;
$Description =~ s/<p>//isg;
$Keywords =~ s/
//isg;
$Keywords =~ s,\n,,g;
$Keywords =~ s/<p>//isg;
$startmain = time();
use LWP::UserAgent;
use HTTP::Request;
use HTTP::Headers;
$url = $URL;
my ($ua,$req,$res);
$ua = LWP::UserAgent->new();
$ua->timeout(333);
$useragent = "URLScope Robot for subsites (urlscope\@lookhard.hypermart.net; http://lookhard.hypermart.net/look-bin/Look/spider.cgi)";
$ua->agent($useragent);
$req = new HTTP::Request 'GET' => $url;
$res = $ua->request($req);
$mes = $ua->request($req);
$Title = substr($Title,0,50);
if (length($Title) > 49) {
$Title =~ s/\s+\S*$/../;
}
$Description = substr($Description,0,225);
if (length($Description) > 224) {
$Description =~ s/\s+\S*$/.../;
}
$Keywords = $res->header('X-Meta-Keywords');
$Keywords =~ s/
//isg;
$Keywords =~ s/<p>//isg;
$Keywords =~ s,\n,,g;
$Keywords = substr($Keywords,0,145);
if (length($Keywords) > 144) {
$Keywords =~ s/\s+\S*$/.../;
}
use HTTP::Date;
$lastupd = time2str($res->last_modified);
$endmain = time() - $startmain;
# Update the counter.
open (ID, "<$db_spider_id_file_name") or &cgierr("error in process_form. unable to open id file: $db_spider_id_file_name. Reason: $!");
$ID = <ID> + 1; # Get next ID number
close ID;
# Update the counter.
open (ID, ">$db_spider_id_file_name") or &cgierr("error in get_defaults. unable to open id file: $db_links_id_file_name. Reason: $!");
flock(ID, 2) unless (!$db_use_flock);
print ID $ID; # update counter.
close ID; # automatically removes file lock
open (SPIDER, ">>$db_spider_name") or &cgierr ("Can't open for output counter file. Reason: $!");
if ($db_use_flock) { flock (SPIDER, 2) or &cgierr ("Can't get file lock. Reason: $!"); }
print SPIDER "$ID%%$Title%%$URL%%$Description%%$Keywords%%$Size%%$lastupd\n";
close SPIDER;
open (SPED, ">>$db_spidered_name") or &cgierr ("Can't open for output counter file. Reason: $!");
if ($db_use_flock) { flock (SPED, 2) or &cgierr ("Can't get file lock. Reason: $!"); }
print SPED "$URL\n";
close SPED;
&send_email();
&send_useremail();
return <<"EOM";
<p>
<b>Main Site - Took $endmain seconds to spider the page.</b>
<a href="$wURL"><b>$Title</b></a> - $Description
Keywords: $Keywords
<i><a href="$wURL">$URL</a><small> - size $Size - added on $lastupd</small></i><p>
<b>Sub-Sites - Sites that were crawled off the main page.</b><p>
EOM
}
sub webEncode {
$_ = shift;
return $_ unless m!(\%|\+|\&|\s)!;
s!\%!\%!g;
s!\+!\+!g;
s!\&!\&!g;
s! !\+!g;
return $_;
}
sub StripIgnoreWords {
$_ = shift;
foreach $Ignore (@IgnoredWords) {
s! $Ignore ! !g;
}
return $_;
}
sub send_email {
# --------------------------------------------------------
$mySize1= &filesize($ByteSize);
# Sends an email to the admin, letting him know that there is
# a new link waiting to be validated. No error checking as we don't
# want users to see the informative &cgierr output.
# Set date variable to today's date.
$in{$db_cols[$db_modified]} = &get_date;
# Check to make sure that there is an admin email address defined.
$db_admin_email or &cgierr("Admin Email Address Not Defined in config file!");
my $to = $db_admin_email;
my $from = $FORM{'email'};
my $subject = "Url crawled and put into database: $Title on $Month - $DD - $YYYY\n";
my $msg = qq|
The following url has been fetched
ID: $ID
Title: $Title
URL: $URL
Keywords: $Keywords
Size: $mySize1
Description: $Description
Referer: $ENV{'HTTP_REFERER'}
Remote Address: $ENV{'REMOTE_ADDR'}
Browser and OS: $ENV{'HTTP_USER_AGENT'}
Sincerely,
Links Manager.
|;
# Then mail it away!
require "$db_lib_path/Mailer.pm";
my $mailer = new Mailer ( { smtp => $db_smtp_server,
sendmail => $db_mail_path,
from => $from,
subject => $subject,
to => $to,
msg => $msg,
log => $db_mailer_log
} ) or return;
$mailer->send or return;
}
sub send_useremail {
# --------------------------------------------------------
$mySize2= &filesize($ByteSize);
# Sends an email to the admin, letting him know that there is
# a new link waiting to be validated. No error checking as we don't
# want users to see the informative &cgierr output.
# Set date variable to today's date.
$in{$db_cols[$db_modified]} = &get_date;
# Check to make sure that there is an admin email address defined.
$db_admin_email or &cgierr("Admin Email Address Not Defined in config file!");
my $to = $FORM{'email'};
my $from = $db_admin_email;
my $subject = "Your Url has been crawled and put into database: $Title on $Month - $DD - $YYYY\n";
my $msg = qq|
The following url has been fetched
ID: $ID
Title: $Title
URL: $URL
Keywords: $Keywords
Size: $mySize2
Description: $Description
Referer: $ENV{'HTTP_REFERER'}
Remote Address: $ENV{'REMOTE_ADDR'}
Browser and OS: $ENV{'HTTP_USER_AGENT'}
Sincerely,
Links Manager.
|;
# Then mail it away!
require "$db_lib_path/Mailer.pm";
my $mailer = new Mailer ( { smtp => $db_smtp_server,
sendmail => $db_mail_path,
from => $from,
subject => $subject,
to => $to,
msg => $msg,
log => $db_mailer_log
} ) or return;
$mailer->send or return;
}
sub error {
print "<center>\n";
print "$E\n\n";
print <<"EOM";
</center>
</body>
</html>
EOM
exit;
}
exit;
***END****
