database - 是否有将现有 log4j 日志文件转储到关系数据库的实用程序？

Question

似乎，像一个非常基本的东西，但我找不到它。

我有一堆 log4j/log4net 日志文件。我想将它们转储到数据库中，以便能够轻松分析它们。

我以为我会很快找到一个工具来做到这一点，显然我错了。

有人知道这样的工具吗？

Answer 1

好的，所以我发现没有实用程序。不得不自己写。当然，它是严格根据我的迫切需求量身定制的（时间就是金钱），但是，如果有需要，它可以为您节省一些开始自己的时间。以下是 C# 中的完整代码：

using System;
using System.Collections;
using System.Collections.Generic;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Text.RegularExpressions;

namespace ConsoleApplication3
{
    class Program
    {
        public class LogEntry
        {
            private const string PATTERN = @"^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d\.\d{4}) (\S+) \[(\d+)\] (\w+) (\S+) - (.*)$";
            private static readonly Regex s_regex = new Regex(PATTERN, RegexOptions.Compiled);

            public DateTime TS;
            public string Machine;
            public int Thread;
            public string Level;
            public string Logger;
            public string Message;

            public static LogEntry TryCreate(string line)
            {
                var match = s_regex.Match(line);
                return match.Success ? new LogEntry
                {
                    TS = DateTime.ParseExact(match.Groups[1].Value, "yyyy-MM-dd HH:mm:ss.ffff", CultureInfo.InvariantCulture),
                    Machine = match.Groups[2].Value,
                    Thread = int.Parse(match.Groups[3].Value),
                    Level = match.Groups[4].Value,
                    Logger = match.Groups[5].Value,
                    Message = match.Groups[6].Value,
                } : null;
            }

            public void AppendToMessage(string line)
            {
                Message += Environment.NewLine + line;
            }
        }
        static void Main()
        {
            const string SQL = @"
INSERT INTO log ( ts,  machine,  thread,  level,  logger,  message,  journalId)
         VALUES (@ts, @machine, @thread, @level, @logger, @message, @journalId)
";
            using (var connection = new SqlConnection("server=localhost;database=misc;uid=SantaClaus;pwd=MerryChristmas"))
            {
                connection.Open();
                using (var command = new SqlCommand(SQL, connection))
                {
                    var tsParam = new SqlParameter("@ts", SqlDbType.DateTime);
                    var machineParam = new SqlParameter("@machine", SqlDbType.NVarChar, 32);
                    var threadParam = new SqlParameter("@thread", SqlDbType.Int);
                    var levelParam = new SqlParameter("@level", SqlDbType.NVarChar, 10);
                    var loggerParam = new SqlParameter("@logger", SqlDbType.NVarChar, 128);
                    var messageParam = new SqlParameter("@message", SqlDbType.NVarChar, -1);
                    var journalIdParam = new SqlParameter("@journalId", SqlDbType.Int);
                    command.Parameters.Add(tsParam);
                    command.Parameters.Add(machineParam);
                    command.Parameters.Add(threadParam);
                    command.Parameters.Add(levelParam);
                    command.Parameters.Add(loggerParam);
                    command.Parameters.Add(messageParam);
                    command.Parameters.Add(journalIdParam);

                    // Call Prepare after setting the Commandtext and Parameters.
                    command.Prepare();
                    int i = 0;
                    foreach (var file in Directory.GetFiles(@"c:\tmp\dfbje01"))
                    {
                        journalIdParam.Value = OpenJournal(connection, file);

                        command.Transaction = connection.BeginTransaction();
                        foreach (var e in GetLogEntries(file))
                        {
                            tsParam.Value = e.TS;
                            machineParam.Value = e.Machine;
                            threadParam.Value = e.Thread;
                            levelParam.Value = e.Level;
                            loggerParam.Value = e.Logger;
                            messageParam.Value = e.Message;
                            command.ExecuteNonQuery();
                            ++i;
                            if (i == 1000)
                            {
                                i = 0;
                                command.Transaction.Commit();
                                command.Transaction = connection.BeginTransaction();
                            }
                        }

                        command.Transaction.Commit();
                        CloseJournal(connection, journalIdParam.Value);
                    }
                }
            }
        }

        private static void CloseJournal(SqlConnection connection, object id)
        {
            const string SQL = "UPDATE journal SET done = 1 WHERE id = @id";
            using (var command = new SqlCommand(SQL, connection))
            {
                command.Parameters.Add(new SqlParameter("@id", id));
                command.ExecuteNonQuery();
            }
        }

        private static object OpenJournal(SqlConnection connection, string filePath)
        {
            const string SQL = "INSERT INTO journal (filePath) OUTPUT inserted.id VALUES (@filePath)";
            using (var command = new SqlCommand(SQL, connection))
            {
                command.Parameters.Add(new SqlParameter("@filePath", filePath));
                return command.ExecuteScalar();
            }
        }

        private static IEnumerable<LogEntry> GetLogEntries(string filePath)
        {
            LogEntry prev = null;
            foreach (var line in File.ReadLines(filePath))
            {
                var logEntry = LogEntry.TryCreate(line);
                if (logEntry != null)
                {
                    if (prev != null)
                    {
                        yield return prev;
                    }
                    prev = logEntry;
                }
                else if (prev != null)
                {
                    prev.AppendToMessage(line);
                }
                else
                {
                    // Oops
                    Console.WriteLine(line);
                }
            }
            if (prev != null)
            {
                yield return prev;
            }
        }
    }
}

Answer 2

介意尝试一下最新的 Chainsaw 开发者快照的过滤、搜索、着色功能吗？它有很多特性可以避免使用数据库。如果您使用 VFSLogFilePatternReceiver，它可以解析和跟踪任何常规文本文件，包括那些由 log4net 创建的文件。

可在此处获得 Chainsaw 的最新开发人员快照：http: //people.apache.org/~sdeboy