solr - 带 SOLR 的 NCrawler

Question

我设法使用 NCrawler 抓取了一个网站。是否可以将该数据导入 SOLR，以便我可以使用 SOLR 中的索引数据进行搜索？

如果可能的话，我怎样才能将爬取的数据推送到 SOLR？任何帮助将不胜感激。

提前致谢。

Answer 1

是的，可以将爬取的数据索引到 Solr。我以前做过这个。您将需要创建一个实现 IPipelineStep 的自定义管道步骤并将其添加到您的 NCrawler 实现中。我使用SolrNet作为连接到 Solr 的客户端。

这是一些可以帮助您入门的代码。

 SolrNet.Startup.Init<IndexItem>("http://localhost:8983/solr");

 using(Crawler c = new Crawler("http://ncrawler.codeplex.com/", 
    new HtmlDocumentProcessor(), new AddCrawledItemToSolrIndex()))
 {
     c.ThreadCount = 3;
     c.MaxCrawlDepth = 2;
     c.ExcludeFilter = new[] { new RegexFilter(
        new Regex(@"(\.jpg|\.css|\.js|\.gif|\.jpeg|\.png|\.ico)",
            RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)) },
     c.Crawl();
 }

自定义 IPipelineStep

using System;
using System.Collections.ObjectModel;
using Microsoft.Practices.ServiceLocation;
using MyCrawler.Index;
using NCrawler;
using NCrawler.Interfaces;
using SolrNet;

namespace MyCrawler.Crawler
{
    public class AddCrawledItemToSolrIndex : IPipelineStep
    {
        public void Process(NCrawler.Crawler crawler, PropertyBag propertyBag)
        {
            if (string.IsNullOrWhiteSpace(propertyBag.Text))
                return;

            var indexItem = new IndexItem
            {
                Id = propertyBag.Step.Uri.ToString(),
                Url = propertyBag.Step.Uri.ToString(),
                Host = propertyBag.Step.Uri.Host,
                Content = propertyBag.Text,
                Title = propertyBag.Title,
                LastModified = Convert.ToInt64(DateTimeToUnixTimestamp(propertyBag.LastModified)),
                Date = propertyBag.LastModified.ToString("yyyyMMdd"),
                Keywords = ExtractKeywords(propertyBag.Headers),
                Type = SplitString(propertyBag.ContentType, ';'),
                Digest = CreateMD5Hash(propertyBag.Text),
            };
            var solr = ServiceLocator.Current.GetInstance<ISolrOperations<IndexItem>>();
            solr.Add(indexItem, new AddParameters {CommitWithin = 10000});
        }

        private Collection<string> SplitString(string input, char splitOn)
        {
            var values = input.Split(splitOn);
            var valueCollection = new Collection<string>();
            if (values.Length == 0) return valueCollection;
            foreach (var value in values)
            {
                valueCollection.Add(value.Trim());
            }

            return valueCollection;

        } 

        private double DateTimeToUnixTimestamp(DateTime dateTime)
        {
            return (dateTime - new DateTime(1970, 1, 1).ToLocalTime()).TotalSeconds;
        }

        private string CreateMD5Hash(string input)
        {
            // Use input string to calculate MD5 hash
            var md5 = MD5.Create();
            var inputBytes = Encoding.ASCII.GetBytes(input);
            var hashBytes = md5.ComputeHash(inputBytes);

            // Convert the byte array to hexadecimal string
            var sb = new StringBuilder();
            for (int i = 0; i < hashBytes.Length; i++)
            {
                //sb.Append(hashBytes[i].ToString("X2"));
                // To force the hex string to lower-case letters instead of
                // upper-case, use he following line instead:
                sb.Append(hashBytes[i].ToString("x2")); 
            }
            return sb.ToString();
        }


        private Collection<string> ExtractKeywords(System.Net.WebHeaderCollection headers)
        {
            var keywords = headers["keywords"];
            if (string.IsNullOrWhiteSpace(keywords))
            {
                return new Collection<string>();
            }

            return SplitString(keywords, ',');
        }
    }
}

这是使用以下 IndexItem.cs 类映射到 Solr 索引字段。

using System.Collections.ObjectModel;
using SolrNet.Attributes;

namespace MyCrawler.Index
{
    public class IndexItem
    {
        [SolrField("id")]
        public string Id { get; set; }
        [SolrField("url")]
        public string Url { get; set; }
        [SolrField("host")]
        public string Host { get; set; }
        [SolrField("content")]
        public string Content { get; set; }
        [SolrField("title")]
        public string Title { get; set; }
        [SolrField("description")]
        public string Description { get; set; }
        [SolrField("digest")]
        public string Digest { get; set; }
        [SolrField("keywords")]
        public Collection<string> Keywords { get; set; }
        [SolrField("date")]
        public string Date { get; set; }
        [SolrField("contentLength")]
        public long ContentLength { get; set; }
        [SolrField("lastModified")]
        public long LastModified { get; set; }
        [SolrField("type")]
        public Collection<string> Type { get; set; }
    }
}

Solr 字段定义 (schema.xml) 取自 Nutch 代码库。

  <!-- core fields -->
  <field name="segment" type="string" stored="true" indexed="false"/>
  <field name="digest" type="string" stored="true" indexed="false"/>
  <field name="boost" type="float" stored="true" indexed="false"/>

  <!-- meta-tag fields -->
  <field name="keywords" type="text_general" stored="true" indexed="true" multiValued="true"/>
  <field name="description" type="text_general" stored="true" indexed="true"/>

  <!-- fields for index-basic plugin -->
  <field name="host" type="url" stored="false" indexed="true"/>
  <field name="site" type="string" stored="true" indexed="true"/>
  <field name="url" type="url" stored="true" indexed="true"
      required="true"/>
  <field name="content" type="text_general" stored="true" indexed="true"/>
  <field name="title" type="text_general" stored="true" indexed="true"/>
  <field name="cache" type="string" stored="true" indexed="false"/>
  <field name="tstamp" type="long" stored="true" indexed="true"/>

  <!-- fields for index-anchor plugin -->
  <field name="anchor" type="string" stored="true" indexed="true"
      multiValued="true"/>

  <!-- fields for index-more plugin -->
  <field name="type" type="string" stored="true" indexed="true"
      multiValued="true"/>
  <field name="contentLength" type="long" stored="true"
      indexed="false"/>
  <field name="lastModified" type="long" stored="true"
      indexed="true"/>
  <field name="date" type="string" stored="true" indexed="true"/>

  <!-- fields for languageidentifier plugin -->
  <field name="lang" type="string" stored="true" indexed="true"/>

  <!-- fields for subcollection plugin -->
  <field name="subcollection" type="string" stored="true"
      indexed="true"/>

  <!-- fields for feed plugin -->
  <field name="author" type="string" stored="true" indexed="true"/>
  <field name="tag" type="string" stored="true" indexed="true"/>
  <field name="feed" type="string" stored="true" indexed="true"/>
  <field name="publishedDate" type="string" stored="true"
      indexed="true"/>
  <field name="updatedDate" type="string" stored="true"
      indexed="true"/>

  <!-- catchall field, containing all other searchable text fields (implemented
  via copyField further on in this schema  -->
  <field name="text" type="text_general" indexed="true" stored="false" multiValued="true"/>

  <field name="_version_" type="long" indexed="true" stored="true"/>

  <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
</fields>

显然，您将希望对其进行修改以满足您的需求，并且它可能会使用一些性能改进。但应该是一个很好的参考点。