我一直在使用以下脚本为我的客户网站创建站点地图。问题是它不适用于每个站点。我发现,godaddy 上托管的许多(如果不是全部)网站都没有蜘蛛。如果有人可以在我的脚本中看到错误或知道导致错误的原因,我将非常感谢您的帮助。
提前致谢
set_time_limit(0);
class spider_man
{
var $url;
var $limit;
var $cache;
var $crawled;
var $banned_ext;
var $domain;
function spider_man( $url, $banned_ext, $limit ){
$this->domain = $url;
$this->url = 'http://'.$url ;
$this->banned_ext = $banned_ext ;
$this->limit = $limit ;
if( !fopen( $this->url, "r") ) return false;
else $this->_spider($this->url);
}
function _spider( $url ){
$this->cache = @file_get_contents( urldecode( $url ) );
if( !$this->cache ) return false;
$this->crawled[] = urldecode( $url ) ;
preg_match_all( "#href=\"(https?://[&=a-zA-Z0-9-_./]+)\"#si", $this->cache, $links );
if ( $links ) :
foreach ( $links[1] as $hyperlink ){
if(strpos($hyperlink,$this->domain)===false){ break; }
else{
$this->limit--;
if( ! $this->limit ) return;
if( $this->is_valid_ext( trim( $hyperlink ) ) and !$this->is_crawled( $hyperlink ) ) :
$this->crawled[] = $hyperlink;
echo "Crawling $hyperlink<br />\n";
unset( $this->cache );
$this->_spider( $hyperlink );
endif;
}
}
endif;
}
function is_valid_ext( $url ){
foreach( $this->banned_ext as $ext ){
if( $ext == substr( $url, strlen($url) - strlen( $ext ) ) ) return false;
}
return true;
}
function is_crawled( $url ){
return in_array( $url, $this->crawled );
}
}
$banned_ext = array(".dtd",".css",".xml",".js",".gif",".jpg",".jpeg",".bmp",".ico",".rss",".pdf",".png",".psd",".aspx",".jsp",".srf",".cgi",".exe",".cfm");
$spider = new spider_man( 'domain.com', $banned_ext, 100 );
print_r( $spider->crawled );