3

我试图开始制作网络爬虫。进展顺利，直到我遇到了我无法理解的困惑。我写了以下代码：

我http://www.google.com作为字符串传递URL

public void crawlURL(string URL, string depth)
{
    if (!checkPageHasBeenCrawled(URL))
    {
        PageContent = getURLContent(URL);
        MatchCollection matches = Regex.Matches(PageContent, "href=\"", RegexOptions.IgnoreCase);
        int count = matches.Count;
    }
} 

private string getURLContent(string URL)
{
    string content;
    HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(URL);
    request.UserAgent = "Fetching contents Data";
    WebResponse response = request.GetResponse();
    Stream stream = response.GetResponseStream();

    StreamReader reader = new StreamReader(stream);
    content = reader.ReadToEnd();

    reader.Close();
    stream.Close();
    return content;
}

问题：我正在尝试获取页面的所有链接（http://www.google.com 或任何其他网站），但我看到来自 Regex 匹配项的链接数量较少。当我手动检查源代码中的单词“href=”时，它给了我 19 个链接计数，它给了我 41 次出现。我不明白为什么它让我从代码中减少了单词的数量。

4

1 回答 1

0

我修复并测试了您的正则表达式模式。以下应该更有效地工作。它从 google.ca 获得 11 个匹配项

public void crawlURL(string URL)
        {

            PageContent = getURLContent(URL);
            MatchCollection matches = Regex.Matches(PageContent, "(href=\"https?://[a-z0-9-._~:/?#\\[\\]@!$&'()*+,;=]+(?=\"|$))", RegexOptions.IgnoreCase);
            foreach (Match match in matches)
                Console.WriteLine(match.Value);

            int count = matches.Count;

        }

        private string getURLContent(string URL)
        {
            string content;
            HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(URL);
            request.UserAgent = "Fetching contents Data";
            WebResponse response = request.GetResponse();
            Stream stream = response.GetResponseStream();

            StreamReader reader = new StreamReader(stream);
            content = reader.ReadToEnd();

            reader.Close();
            stream.Close();
            return content;
        }
于 2012-12-12T07:54:00.270 回答