BB.Net / ramblings / posts / erlang basic distributed application

Erlang with OTP is a fairly powerful framework for creating distributed redundant applications. The basic gen_server behavior can easily extended to create a redundant server with built in failover. With Mnesia you also get a replicated Database.

I've been trying to figure out how exactly this is supposed to work, so I've been working on a quick application to demonstrate this. It's nothing fancy, just a simple set(k,v) and get(k) API.

The files are available as a tarball from here ddict.tgz

-module(ddict).
-behaviour(gen_server).
-export([start/0,stop/0,terminate/2]).
-export([init/1, handle_call/3, handle_cast/2,handle_info/2]).
-export([create_schema/0]).
-export([get/1,set/2]).


-define(GD,{global, ddict}).

-include_lib("stdlib/include/qlc.hrl").
-record(rec, {key, value}).


init_mnesia() ->
    mnesia:start(),
    ok = mnesia:wait_for_tables([rec], 2000).

init(_Arg) ->
    process_flag(trap_exit, true),
    io:format("dict server starting~n"),
    init_mnesia(),
    {ok, []}.

start() ->
    gen_server:start_link(?GD, ddict, [], []).

stop() ->
    gen_server:cast(?GD, stop).

terminate(Reason, State) ->
   io:format("dict server terminating~n").

%"model" methods
do_get(Key) ->
    Res = mnesia:dirty_read({rec, Key}),
    case Res of 
        [] -> undefined;
        [Rec] -> Rec#rec.value
    end.

do_set(Key, Value) ->
    F = fun() ->
            Row = #rec{key=Key, value=Value},
            mnesia:write(Row)
        end,
    {atomic, ok} = mnesia:transaction(F),
    ok.

%"controller" methods
handle_call({get, Key}, From, State) ->
    Rec = do_get(Key),
    {reply, Rec, State};

handle_call({set, Key, Value}, From, State) ->
    Rec = do_set(Key, Value),
    {reply, Rec, State}.

handle_cast(stop, State) ->
    io:format("ddict server stopping~n"),
    {stop, normal, State}.

handle_info(Info, State) ->
    {noreply, State}.


%"client api" methods
get(Key) ->
   gen_server:call(?GD, {get, Key}).

set(Key,Value) ->
   gen_server:call(?GD, {set, Key, Value}).


create_schema() ->
    mnesia:create_schema([node()|nodes()]),
    mnesia:start(),
    %this is defnitely wrong
    lists:foreach(fun(N) ->
        io:format("starting mnesia on ~w~n", [N]),
        rpc:call(N, mnesia, start, [])
    end, nodes()),
    mnesia:create_table(rec, [
        {disc_copies, [node()|nodes()]},
        {attributes, record_info(fields, rec)}
    ]).
download file "main gen_server file"

The key to this server being distributed is the use of {global, ddict} as the server name, instead of {local, ddict}. This enables other nodes in the cluster to see this server.

do_get() and do_set() are the "model" like methods that deal with mnesia. handle_call defines the gen_server api. get() and set() are helper functions that call the remote gen server. If there was more to this module, it would be a good idea to put these methods in separate modules.

The one thing I am not sure about is the create_schema() method. I'm sure there is a propper way to initalize mnesia on a cluster, I just have no idea what it is yet :-)

To make this into a propper gen server the supervisor and application needs to be defined with the following three files:

-module(ddict_sup).
-behaviour(supervisor).

-export([start_link/0]).
-export([init/1]).

start_link() ->
    supervisor:start_link(ddict_sup, []).

init(_Args) ->
    {ok, {{one_for_one, 10, 60},
          [{ddict, {ddict, start, []},
            permanent, brutal_kill, worker, [ddict]}]}}.
download file "/ramblings/files/erlang/ddict/ddict_sup.erl"
-module(ddict_app).
-behaviour(application).

-export([start/2, stop/1,go/0]).

start(_Type, _Args) ->
    ddict_sup:start_link().

stop(_State) ->
    io:format("ddict server terminating~n"),
    ok.

go() ->
    application:start(ddict).
download file "/ramblings/files/erlang/ddict/ddict_app.erl"
{application, ddict,
[
    {mod, {ddict_app,[]}}
]}.
download file "/ramblings/files/erlang/ddict/ddict.app"

To get erlang to start this application on boot, a config file for each node needs to be written:

[{kernel,
  [{distributed, [{ddict, 3000, [one@media, {two@media}]}]},
   {sync_nodes_optional, [two@media]},
   {sync_nodes_timeout, 5000}
  ]
 }
].
download file "/ramblings/files/erlang/ddict/one.config"
[{kernel,
  [{distributed, [{ddict, 3000, [one@media, {two@media}]}]},
   {sync_nodes_optional, [one@media]},
   {sync_nodes_timeout, 5000}
  ]
 }
].
download file "/ramblings/files/erlang/ddict/two.config"

To create the initial database I ran the ddict:create_schema method, which I'm sure is completely incorrect, but it works:

erl -sname one -config one.config
erl -sname two -config two.config

(one@media)1> ddict:create_schema().
starting mnesia on two@media
{atomic,ok}
(one@media)2> mnesia:info().
...
running db nodes   = [two@media,one@media]
disc_copies        = [rec,schema]
[{one@media,disc_copies},{two@media,disc_copies}] = [schema,rec]
...
ok
download file "/ramblings/files/erlang/ddict/create_db.txt"

Once that is done, the application can be started with

erl  -pa . -sname one -config one.config -s ddict_app go
erl  -pa . -sname two -config two.config -s mnesia start -s ddict_app go

I have to start mnesia separately on the second VM because I haven't yet figured out how mnesia should be started when dealing with distributed applications. mnesia needs to be running on both nodes, but not the ddict application itself.

Once it is running, you can call ddict:set("Foo","bar") and ddict:get("Foo"). You can also kill either VM, and it will restart the server after 3 seconds on the other node.

Comments here