4

我正在评估使用 Erlang ETS 来存储一个大型内存数据集。我的测试数据源是一个 CSV 文件，它只消耗 350 MB 的磁盘。

我的解析器逐行读取并将其拼接成一个列表，然后使用“bag”配置创建一个元组并将其存储在ETS中。

在 ETS 中加载所有数据后，我注意到我的计算机的 8GB RAM 全部消失了，并且操作系统创建了虚拟内存，占用了接近 16GB 或 RAM 的某个地方。erlang 的 Beam 进程消耗的内存似乎是磁盘数据大小的10 倍。

这是测试代码：

-module(load_test_data).
-author("gextra").

%% API
-export([test/0]).

init_ets() ->
  ets:new(memdatabase, [bag, named_table]).

parse(File) ->
  {ok, F} = file:open(File, [read, raw]),
  parse(F, file:read_line(F), []).

parse(F, eof, Done) ->
  file:close(F),
  lists:reverse(Done);

parse(F, Line, Done) ->
  parse(F, file:read_line(F), [ parse_row_commodity_data(Line) | Done ]).

parse_row_commodity_data(Line) ->
  {ok, Data} = Line,
  %%io:fwrite(Data),
  LineList          = re:split(Data,"\,",[{return,list}]),
  ReportingCountry  = lists:nth(1, LineList),
  YearPeriod        = lists:nth(2, LineList),
  Year              = lists:nth(3, LineList),
  Period            = lists:nth(4, LineList),
  TradeFlow         = lists:nth(5, LineList), 
  Commodity         = lists:nth(6, LineList),
  PartnerCountry    = lists:nth(7, LineList),
  NetWeight         = lists:nth(8, LineList),
  Value             = lists:nth(9, LineList),
  IsReported        = lists:nth(10, LineList),
  ets:insert(memdatabase, {YearPeriod ++ ReportingCountry ++ Commodity , { ReportingCountry, Year, Period, TradeFlow, Commodity, PartnerCountry, NetWeight, Value, IsReported } }).


test() ->
  init_ets(),
  parse("/data/000-2010-1.csv").
4

1 回答 1

4

它在很大程度上取决于您将其拼接到列表中的含义，然后创建一个 tuple。特别是拼接到列表中会占用大量内存。如果拆分成列表，一个字节可以占用 16B。5.6GB 很简单。

编辑：

试试这个：

parse(File) ->
  {ok, F} = file:open(File, [read, raw, binary]),
  ok = parse(F, binary:compile_pattern([<<$,>>, <<$\n>>])),
  ok = file:close(F).

parse(F, CP) ->
  case file:read_line(F) of
    {ok, Line} ->
      parse_row_commodity_data(Line, CP),
      parse(F, CP);
    eof -> ok
  end.

parse_row_commodity_data(Line, CP) ->
  [ ReportingCountry, YearPeriod, Year, Period, TradeFlow, Commodity,
    PartnerCountry, NetWeight, Value, IsReported]
      = binary:split(Line, CP, [global, trim]),
  true = ets:insert(memdatabase, {
         {YearPeriod, ReportingCountry, Commodity},
         { ReportingCountry, Year, Period, TradeFlow, Commodity,
           PartnerCountry, NetWeight, Value, IsReported}
       }).
于 2014-02-22T21:24:04.773 回答