Mnesia secondary index: a memory leak or lasy cleanup?

classic Classic list List threaded Threaded
3 messages Options
Reply | Threaded
Open this post in threaded view
|

Mnesia secondary index: a memory leak or lasy cleanup?

Karolis Petrauskas-2
Hello,

I have strange trend in memory consumption on some servers. The ETS
memory constantly increases. The tables that are increasing are
mnesia_index tables. It looks like these tables are not cleaned-up
after the corresponding records are deleted from the indexed table.
Maybe the index is cleaned up later? I was waiting for a month, the
mnesia_index table has grown to 1GB approximately.

To reproduce the situation I made a simple test case. The code of the
tst module is shown at the end of this email.

First I create a bag table and add some (1000) records to it:

    4> tst:create_table().
    ok
    10> tst:data_add().
    ok
    12> ets:i().
     id              name              type  size   mem      owner
     ----------------------------------------------------------------------------
     ...
     45084           mnesia_index      ordered_set 1000   10088
mnesia_monitor
     ...
     t               t                 bag   1000   9298     mnesia_monitor
    ok

The table and the index has same number of records (the data for the
primary and the secondary indexes are unique in this case).
Then I delete the records:

    14> tst:data_del().
    ok
    16> ets:i().
     id              name              type  size   mem      owner
     ----------------------------------------------------------------------------
     ...
     45084           mnesia_index      ordered_set 1000   10088
mnesia_monitor
     ...
     t               t                 bag   0      298      mnesia_monitor
    ok

Here the table has 0 records (as expected), and the index still has
1000 records.

The code of the tst module:

    -module(tst).
    -export([create_table/0, data_add/0, data_del/0]).
    -record(t, {a, b, c}).

    create_table() ->
        {atomic, ok} = mnesia:create_table(t, [{type, bag},
{attributes, record_info(fields, t)}]),
        {atomic, ok} = mnesia:add_table_index(t, b),
        ok.

    data_add() ->
        mnesia:activity(transaction, fun () ->
            lists:foreach(fun (Key) ->
                mnesia:write(#t{a = Key, b = Key})
            end, lists:seq(1, 1000))
        end).

    data_del() ->
        mnesia:activity(transaction, fun () ->
            lists:foreach(fun (Key) ->
                [R] = mnesia:match_object(#t{a = Key, _ = '_'}),
                mnesia:delete_object(R)
            end, lists:seq(1, 1000))
        end).


Karolis Petrauskas
_______________________________________________
erlang-questions mailing list
[hidden email]
http://erlang.org/mailman/listinfo/erlang-questions
Reply | Threaded
Open this post in threaded view
|

Re: Mnesia secondary index: a memory leak or lasy cleanup?

Karolis Petrauskas-2
Forgot to mention the versions.
This case is reproducible with "Erlang/OTP 21 [erts-10.0] [source]
[64-bit] [smp:8:8] [ds:8:8:10] [async-threads:1] [hipe]".
The same behaviour is observed on Erlang/OTP 19.

Karolis Petrauskas
On Wed, Aug 15, 2018 at 5:49 PM Karolis Petrauskas
<[hidden email]> wrote:

>
> Hello,
>
> I have strange trend in memory consumption on some servers. The ETS
> memory constantly increases. The tables that are increasing are
> mnesia_index tables. It looks like these tables are not cleaned-up
> after the corresponding records are deleted from the indexed table.
> Maybe the index is cleaned up later? I was waiting for a month, the
> mnesia_index table has grown to 1GB approximately.
>
> To reproduce the situation I made a simple test case. The code of the
> tst module is shown at the end of this email.
>
> First I create a bag table and add some (1000) records to it:
>
>     4> tst:create_table().
>     ok
>     10> tst:data_add().
>     ok
>     12> ets:i().
>      id              name              type  size   mem      owner
>      ----------------------------------------------------------------------------
>      ...
>      45084           mnesia_index      ordered_set 1000   10088
> mnesia_monitor
>      ...
>      t               t                 bag   1000   9298     mnesia_monitor
>     ok
>
> The table and the index has same number of records (the data for the
> primary and the secondary indexes are unique in this case).
> Then I delete the records:
>
>     14> tst:data_del().
>     ok
>     16> ets:i().
>      id              name              type  size   mem      owner
>      ----------------------------------------------------------------------------
>      ...
>      45084           mnesia_index      ordered_set 1000   10088
> mnesia_monitor
>      ...
>      t               t                 bag   0      298      mnesia_monitor
>     ok
>
> Here the table has 0 records (as expected), and the index still has
> 1000 records.
>
> The code of the tst module:
>
>     -module(tst).
>     -export([create_table/0, data_add/0, data_del/0]).
>     -record(t, {a, b, c}).
>
>     create_table() ->
>         {atomic, ok} = mnesia:create_table(t, [{type, bag},
> {attributes, record_info(fields, t)}]),
>         {atomic, ok} = mnesia:add_table_index(t, b),
>         ok.
>
>     data_add() ->
>         mnesia:activity(transaction, fun () ->
>             lists:foreach(fun (Key) ->
>                 mnesia:write(#t{a = Key, b = Key})
>             end, lists:seq(1, 1000))
>         end).
>
>     data_del() ->
>         mnesia:activity(transaction, fun () ->
>             lists:foreach(fun (Key) ->
>                 [R] = mnesia:match_object(#t{a = Key, _ = '_'}),
>                 mnesia:delete_object(R)
>             end, lists:seq(1, 1000))
>         end).
>
>
> Karolis Petrauskas
_______________________________________________
erlang-questions mailing list
[hidden email]
http://erlang.org/mailman/listinfo/erlang-questions
Reply | Threaded
Open this post in threaded view
|

Re: Mnesia secondary index: a memory leak or lasy cleanup?

Dan Gudmundsson-3
Yep seems broken for 'bag' types.

I will take at look at that, thanks for the bug report.

/Dan

On Wed, Aug 15, 2018 at 8:38 PM Karolis Petrauskas <[hidden email]> wrote:
Forgot to mention the versions.
This case is reproducible with "Erlang/OTP 21 [erts-10.0] [source]
[64-bit] [smp:8:8] [ds:8:8:10] [async-threads:1] [hipe]".
The same behaviour is observed on Erlang/OTP 19.

Karolis Petrauskas
On Wed, Aug 15, 2018 at 5:49 PM Karolis Petrauskas
<[hidden email]> wrote:
>
> Hello,
>
> I have strange trend in memory consumption on some servers. The ETS
> memory constantly increases. The tables that are increasing are
> mnesia_index tables. It looks like these tables are not cleaned-up
> after the corresponding records are deleted from the indexed table.
> Maybe the index is cleaned up later? I was waiting for a month, the
> mnesia_index table has grown to 1GB approximately.
>
> To reproduce the situation I made a simple test case. The code of the
> tst module is shown at the end of this email.
>
> First I create a bag table and add some (1000) records to it:
>
>     4> tst:create_table().
>     ok
>     10> tst:data_add().
>     ok
>     12> ets:i().
>      id              name              type  size   mem      owner
>      ----------------------------------------------------------------------------
>      ...
>      45084           mnesia_index      ordered_set 1000   10088
> mnesia_monitor
>      ...
>      t               t                 bag   1000   9298     mnesia_monitor
>     ok
>
> The table and the index has same number of records (the data for the
> primary and the secondary indexes are unique in this case).
> Then I delete the records:
>
>     14> tst:data_del().
>     ok
>     16> ets:i().
>      id              name              type  size   mem      owner
>      ----------------------------------------------------------------------------
>      ...
>      45084           mnesia_index      ordered_set 1000   10088
> mnesia_monitor
>      ...
>      t               t                 bag   0      298      mnesia_monitor
>     ok
>
> Here the table has 0 records (as expected), and the index still has
> 1000 records.
>
> The code of the tst module:
>
>     -module(tst).
>     -export([create_table/0, data_add/0, data_del/0]).
>     -record(t, {a, b, c}).
>
>     create_table() ->
>         {atomic, ok} = mnesia:create_table(t, [{type, bag},
> {attributes, record_info(fields, t)}]),
>         {atomic, ok} = mnesia:add_table_index(t, b),
>         ok.
>
>     data_add() ->
>         mnesia:activity(transaction, fun () ->
>             lists:foreach(fun (Key) ->
>                 mnesia:write(#t{a = Key, b = Key})
>             end, lists:seq(1, 1000))
>         end).
>
>     data_del() ->
>         mnesia:activity(transaction, fun () ->
>             lists:foreach(fun (Key) ->
>                 [R] = mnesia:match_object(#t{a = Key, _ = '_'}),
>                 mnesia:delete_object(R)
>             end, lists:seq(1, 1000))
>         end).
>
>
> Karolis Petrauskas
_______________________________________________
erlang-questions mailing list
[hidden email]
http://erlang.org/mailman/listinfo/erlang-questions

_______________________________________________
erlang-questions mailing list
[hidden email]
http://erlang.org/mailman/listinfo/erlang-questions