1

我想知道 MongoDB 2.2 聚合框架中的 $group 函数是否是多线程的。

对于这个问题，我做了一些小测试。我使用的数据集用于存储大约 400 万封电子邮件，每封电子邮件的格式如下：

shard1:PRIMARY> db.spams.findOne()
{
"IP" : "113.162.134.245",
"_id" : ObjectId("4ebe8c84466e8b1a56000028"),
"attach" : [ ],
"bot" : "Lethic",
"charset" : "iso-8859-1",
"city" : "",
"classA" : "113",
"classB" : "113.162",
"classC" : "113.162.134",
"content_type" : [ ],
"country" : "Vietnam",
"cte" : "7bit",
"date" : ISODate("2011-11-11T00:07:12Z"),
"day" : "2011-11-11",
"from_domain_a" : "domain157939.com",
"geo" : "VN",
"host" : "",
"lang" : "unknown",
"lat" : 16,
"long" : 106,
"sequenceID" : "user648",
"size" : 1060,
"smtp-mail-from_a" : "barriefrancisco@domain157939.com",
"smtp-rcpt-to_a" : "jaunn@domain555065.com",
"subject_ta" : "nxsy8",
"uri" : [ ],
"uri_domain" : [ ],
"x_p0f_detail" : "2000 SP4, XP SP1+",
"x_p0f_genre" : "Windows",
"x_p0f_signature" : "65535:105:1:48:M1402,N,N,S:."
}

我设计了一个查询来查找一天、一周、一个月、半年和一年内的所有电子邮件。然后按“bot”字段对结果进行分组。

我使用聚合框架和 java 驱动来做到这一点。Java代码如下：

public class RangeQuery {
final private String mongoUrl = "172.16.10.61:30000";
final private String databaseName = "test";
final private String collecName = "spams";
private DBCollection collection = null;
private DB db = null;

    public void init(){
    Mongo mongo = null;
    try {
        mongo = new Mongo(new DBAddress(mongoUrl));
    } catch (MongoException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (UnknownHostException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    db = mongo.getDB(databaseName);
    db.requestStart();
    collection = db.getCollection(collecName);
}

    public void queryRange_GroupBot(boolean printResult){
    DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'hh:mm:ss'Z'");
    String toDateStr [] = new String[5] ;
    toDateStr[0] = "2011-01-02T00:00:00Z";
    toDateStr[1] = "2011-01-07T00:00:00Z";
    toDateStr[2] = "2011-02-01T00:00:00Z";
    toDateStr[3] = "2011-06-01T00:00:00Z";
    toDateStr[4] = "2012-01-01T00:00:00Z";

    String toPrint [] = new String[5];
    toPrint[0] = "Within One day";
    toPrint[1] = "Within One week";
    toPrint[2] = "Within One month";
    toPrint[3] = "Within half year";
    toPrint[4] = "Within One year";

    try {
        System.out.println("\n------Query Time Range Group by Bot------");
        for(int i = 0;i < 5;i++){
            System.out.println("    ---" + toPrint[i] + "---");
            Date fromDate = formatter.parse("2011-01-01T00:00:00Z");
            Date toDate = formatter.parse(toDateStr[i]);

            DBObject groupFields = new BasicDBObject( "_id", "$bot");
            groupFields.put("sum", new BasicDBObject( "$sum", 1));
            DBObject group = new BasicDBObject("$group", groupFields);

            DBObject cond1 = new BasicDBObject();
            cond1.put("date", new BasicDBObject("$gte", fromDate));
            DBObject cond2 = new BasicDBObject();
            cond2.put("date", new BasicDBObject("$lte", toDate));
            DBObject match1 = new BasicDBObject("$match", cond1 );
            DBObject match2 = new BasicDBObject("$match", cond2 );

            for(int j = 0;j < 1;j++){
                Long runBefore = Calendar.getInstance().getTime().getTime();
                AggregationOutput aggOutput = collection.aggregate(match1, match2, group);
                Long runAfter = Calendar.getInstance().getTime().getTime();
                if(printResult){
                    System.out.println(aggOutput.getCommandResult());
                }
                System.out.println("[Query Range + Group by Bot]: " + (runAfter - runBefore) + " ms.");
            }
        }
    } catch (ParseException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

    public static void main(String[] args){
    RangeQuery rangQuery = new RangeQuery();
    rangQuery.init();
    rangQuery.queryRange_GroupBot_MapReduce(true);
  }
  }

结果如下所示：

  Within One day(2011-01-01 -> 2011-01-02)      54173 ms
  Within One week(2011-01-01 -> 2011-01-07)     54277 ms
  Within One month(2011-01-01 -> 2011-02-01)    54387 ms
  Within half year(2011-01-01 -> 2011-06-01)    53035 ms
  Within One year(2011-01-01 -> 2012-01-01)     54116 ms

令我惊讶的是，通常一年以上的组应该比一天慢，因为它包含更多记录。（数据集中的记录随时间均匀分布）

如果我只使用 db.spams.find({"date":{$gt:ISODate(xxx), {$lt: xxx}}}).count，我可以看到查询一年比查询一天花费更长的时间。

但是为什么当我使用 $group 时，当我扩大时间范围时，这个函数需要几乎相同的时间？

我知道聚合框架是用 C++ 编写的，我使用的是 mongodb 2.2，聚合框架是否使用了多线程或其他一些方法来提高性能？

4

1 回答 1

2

根据这个讨论：https ://groups.google.com/forum/?fromgroups=#!topic/mongodb-user/xCSww5spXPc

每个管道当前都是单线程的，但您可以并行运行不同的管道。因此，如果您有 100 个连接，每个连接都运行一个聚合命令，那么这些连接可能会并行运行 - 但每个命令将在 1 个线程上运行。

于 2013-01-29T12:05:25.800 回答