我正在编写一个每周运行一次的 php crone 作业脚本
此脚本的主要目的是从 TED we 网站上提供的所有 TED 演讲中获取详细信息（例如，使这个问题更容易理解）
这个脚本运行大约需要 70 分钟，它会运行超过 2000 个网页
我的问题是：
1）是否有更好/更快的方式来每次获取网页，我使用以下功能：
file_get_contents_curl($url)
2) 把所有的谈话放在一个数组中是一个好习惯吗（可能会变得很大）
3) 一般来说，有没有更好的方法可以从网站上获取所有 ted 会谈的详细信息？在 TED 网站上“爬行”以获得所有演讲的最佳方式是什么
**我已经检查了使用 rss 提要的选项，但它缺少我需要的一些细节。
谢谢
<?php
define("START_ID", 1);
define("STOP_TED_QUERY",20);
define ("VALID_PAGE","TED | Talks");
/**
* this script will run as a cron job and will go over all pages
* on TED http://www.ted.com/talks/view/id/
* from id 1 till there are no more pages
*/
/**
* function get a file using curl (fast)
* @param $url - url which we want to get its content
* @return the data of the file
* @author XXXXX
*/
function file_get_contents_curl($url)
{
$ch = curl_init();
curl_setopt($ch, CURLOPT_HEADER, 0);
curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
curl_setopt($ch, CURLOPT_URL, $url);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
$data = curl_exec($ch);
curl_close($ch);
return $data;
}
//will hold all talks in array
$tedTalks = array();
//id to start the query from
$id=START_ID;
//will indicate when needed to stop the query beacuse reached the end id's on TED website
$endOFQuery=0;
//get the time
$time_start = microtime(true);
//start the query on TED website
//if we will query 20 pages in a row that do not exsist we will stop the querys and assume there are no more
while ($endOFQuery < STOP_TED_QUERY){
//get the page of the talk
$html = file_get_contents_curl("http://www.ted.com/talks/view/id/$id");
//parsing begins here:
$doc = new DOMDocument();
@$doc->loadHTML($html);
$nodes = $doc->getElementsByTagName('title');
//get and display what you need:
$title = $nodes->item(0)->nodeValue;
//check if this a valid page
if (! strcmp ($title , VALID_PAGE ))
//this is a removed ted talk or the end of the query so raise a flag (if we get anough of these in a row we will stop)
$endOFQuery++;
else {
//this is a valid TED talk get its details
//reset the flag for end of query
$endOFQuery = 0;
//get meta tags
$metas = $doc->getElementsByTagName('meta');
//get the tag we need (keywords)
for ($i = 0; $i < $metas->length; $i++)
{
$meta = $metas->item($i);
if($meta->getAttribute('name') == 'keywords')
$keywords = $meta->getAttribute('content');
}
//create new talk object and populate it
$talk = new Talk();
//set its ted id from ted web site
$talk->setID($id);
//parse the name (name has un-needed char's in the end)
$talk->setName( substr($title, 0, strpos( $title, '|')) );
//parse the String of tags to array
$keywords = explode(",", $keywords);
//remove un-needed items from it
$keywords=array_diff($keywords, array("TED","Talks"));
//add the filters tags to the talk
$talk->setTags($keywords);
//add to the total talks array
$tedTalks[]=$talk;
}
//move to the next ted talk ID to query
$id++;
} //end of the while
$time_end = microtime(true);
$execution_time = ($time_end - $time_start);
echo "this took (sec) : ".$execution_time;
?>