1

我们使用 RavenDB (2261) 作为基于队列的视频上传系统的后端,并且我们被要求提供有关上传系统的各种指标的“实时”SLA 报告。

文档格式如下所示:

{
  "ClipGuid": "01234567-1234-abcd-efef-123412341234",
  "CustomerId": "ABC123",
  "Title": "Shakespeare in Love",
  "DurationInSeconds": 82,
  "StateChanges": [
    {
      "OldState": "DoesNotExist",
      "NewState": "ReceivedFromUpload",
      "ChangedAt": "2013-03-15T15:38:38.7050002Z"
    },
    {
      "OldState": "ReceivedFromUpload",
      "NewState": "Validating",
      "ChangedAt": "2013-03-15T15:38:38.8453975Z"
    },
    {
      "OldState": "Validating",
      "NewState": "AwaitingSubmission",
      "ChangedAt": "2013-03-15T15:38:39.9529762Z"
    },
    {
      "OldState": "AwaitingSubmission",
      "NewState": "Submitted",
      "ChangedAt": "2013-03-15T15:38:43.4785084Z"
    },
    {
      "OldState": "Submitted",
      "NewState": "Playable",
      "ChangedAt": "2013-03-15T15:41:39.5523223Z"
    }
  ],
}

在每个 ClipInfo 记录中,每次剪辑从处理链的一个部分传递到另一个部分时,都会添加一组 StateChanges。我们需要做的是将这些 StateChanges 减少到两个特定的时间跨度——我们需要知道剪辑从 DoesNotExist 变为 AwaitingSubmission 需要多长时间,以及从 DoesNotExist 变为 Playable 需要多长时间。然后我们需要按日期/时间对这些持续时间进行分组,这样我们就可以绘制一个简单的 SLA 报告,如下所示:

Mockup video upload SLA

The necessary predicates can be expressed as LINQ statements but when I try specifying this sort of complex logic within a Raven query I just seem to get back empty results (or lots of DateTime.MinValue results)

I realise document databases like Raven aren't ideal for reporting - and we're happy to explore replication into SQL or some other sort of caching mechanism - but at the moment I just can't see any way of extracting the data other than doing multiple queries to retrieve the entire contents of the store and then performing the calculations in .NET.

Any recommendations?

Thanks,

Dylan

4

1 回答 1

0

I have made some assumptions which you may need to adjust for:

  • You operate strictly in the UTC time zone - your "day" is midnight to midnight UTC.
  • Your week is Sunday through Saturday
  • The date you want to group by is the first status date reported (the one marked with "DoesNotExist" as its old state.)

You will need a separate map/reduce index per date bracket that you are grouping on - Daily, Weekly, Monthly.

They are almost identical, except for how the starting date is defined. If you want to get creative, you might be able to come up with a way to make these into a generic index definition - but they will always end up being three separate indexes in RavenDB.

// This is the resulting class that all of these indexes will return
public class ClipStats
{
    public int CountClips { get; set; }
    public int NumPassedWithinTwentyPct { get; set; }
    public int NumPlayableWithinOneHour { get; set; }
    public DateTime Starting { get; set; }
}

public class ClipStats_ByDay : AbstractIndexCreationTask<ClipInfo, ClipStats>
{
    public ClipStats_ByDay()
    {
        Map = clips => from clip in clips
                        let state1 = clip.StateChanges.FirstOrDefault(x => x.OldState == "DoesNotExist")
                        let state2 = clip.StateChanges.FirstOrDefault(x => x.NewState == "AwaitingSubmission")
                        let state3 = clip.StateChanges.FirstOrDefault(x => x.NewState == "Playable")
                        let time1 = state2.ChangedAt - state1.ChangedAt
                        let time2 = state3.ChangedAt - state1.ChangedAt
                        select new
                        {
                            CountClips = 1,
                            NumPassedWithinTwentyPct = time1.TotalSeconds < clip.DurationInSeconds * 0.2 ? 1 : 0,
                            NumPlayableWithinOneHour = time2.TotalHours < 1 ? 1 : 0,
                            Starting = state1.ChangedAt.Date
                        };

        Reduce = results => from result in results
                            group result by result.Starting
                            into g
                            select new
                            {
                                CountClips = g.Sum(x => x.CountClips),
                                NumPassedWithinTwentyPct = g.Sum(x => x.NumPassedWithinTwentyPct),
                                NumPlayableWithinOneHour = g.Sum(x => x.NumPlayableWithinOneHour),
                                Starting = g.Key
                            };
    }
}

public class ClipStats_ByWeek : AbstractIndexCreationTask<ClipInfo, ClipStats>
{
    public ClipStats_ByWeek()
    {
        Map = clips => from clip in clips
                        let state1 = clip.StateChanges.FirstOrDefault(x => x.OldState == "DoesNotExist")
                        let state2 = clip.StateChanges.FirstOrDefault(x => x.NewState == "AwaitingSubmission")
                        let state3 = clip.StateChanges.FirstOrDefault(x => x.NewState == "Playable")
                        let time1 = state2.ChangedAt - state1.ChangedAt
                        let time2 = state3.ChangedAt - state1.ChangedAt
                        select new
                        {
                            CountClips = 1,
                            NumPassedWithinTwentyPct = time1.TotalSeconds < clip.DurationInSeconds * 0.2 ? 1 : 0,
                            NumPlayableWithinOneHour = time2.TotalHours < 1 ? 1 : 0,
                            Starting = state1.ChangedAt.Date.AddDays(0 - (int) state1.ChangedAt.Date.DayOfWeek)
                        };

        Reduce = results => from result in results
                            group result by result.Starting
                            into g
                            select new
                            {
                                CountClips = g.Sum(x => x.CountClips),
                                NumPassedWithinTwentyPct = g.Sum(x => x.NumPassedWithinTwentyPct),
                                NumPlayableWithinOneHour = g.Sum(x => x.NumPlayableWithinOneHour),
                                Starting = g.Key
                            };
    }
}

public class ClipStats_ByMonth : AbstractIndexCreationTask<ClipInfo, ClipStats>
{
    public ClipStats_ByMonth()
    {
        Map = clips => from clip in clips
                        let state1 = clip.StateChanges.FirstOrDefault(x => x.OldState == "DoesNotExist")
                        let state2 = clip.StateChanges.FirstOrDefault(x => x.NewState == "AwaitingSubmission")
                        let state3 = clip.StateChanges.FirstOrDefault(x => x.NewState == "Playable")
                        let time1 = state2.ChangedAt - state1.ChangedAt
                        let time2 = state3.ChangedAt - state1.ChangedAt
                        select new
                        {
                            CountClips = 1,
                            NumPassedWithinTwentyPct = time1.TotalSeconds < clip.DurationInSeconds * 0.2 ? 1 : 0,
                            NumPlayableWithinOneHour = time2.TotalHours < 1 ? 1 : 0,
                            Starting = state1.ChangedAt.Date.AddDays(1 - state1.ChangedAt.Date.Day)
                        };

        Reduce = results => from result in results
                            group result by result.Starting
                            into g
                            select new
                            {
                                CountClips = g.Sum(x => x.CountClips),
                                NumPassedWithinTwentyPct = g.Sum(x => x.NumPassedWithinTwentyPct),
                                NumPlayableWithinOneHour = g.Sum(x => x.NumPlayableWithinOneHour),
                                Starting = g.Key
                            };
    }
}

Then when you want to query...

var now = DateTime.UtcNow;

var today = now.Date;
var dailyStats = session.Query<ClipStats, ClipStats_ByDay>()
                        .FirstOrDefault(x => x.Starting == today);

var startOfWeek = today.AddDays(0 - (int) today.DayOfWeek);
var weeklyStats = session.Query<ClipStats, ClipStats_ByWeek>()
                         .FirstOrDefault(x => x.Starting == startOfWeek);

var startOfMonth = today.AddDays(1 - today.Day);
var monthlyStats = session.Query<ClipStats, ClipStats_ByMonth>()
                          .FirstOrDefault(x => x.Starting == startOfMonth);

In the results, you will have totals. So if you want percent averages for your SLA, simply divide the statistic by the count, which is also returned.

于 2013-03-18T22:25:58.443 回答