Add neotoma regression test
Integration regression test that checks for correct generation of .erl files from .peg ones and that they are cleaned up.
diff --git a/inttest/neotoma1/mock/neotoma/priv/peg_includes.hrl b/inttest/neotoma1/mock/neotoma/priv/peg_includes.hrl
new file mode 100644
index 0000000..183b98c
--- /dev/null
+++ b/inttest/neotoma1/mock/neotoma/priv/peg_includes.hrl
@@ -0,0 +1,253 @@
+-file("peg_includes.hrl", 1).
+-type index() :: {{line, pos_integer()}, {column, pos_integer()}}.
+-type input() :: binary().
+-type parse_failure() :: {fail, term()}.
+-type parse_success() :: {term(), input(), index()}.
+-type parse_result() :: parse_failure() | parse_success().
+-type parse_fun() :: fun((input(), index()) -> parse_result()).
+-type xform_fun() :: fun((input(), index()) -> term()).
+
+-spec p(input(), index(), atom(), parse_fun(), xform_fun()) -> parse_result().
+p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
+ case get_memo(StartIndex, Name) of % See if the current reduction is memoized
+ {ok, Memo} -> %Memo; % If it is, return the stored result
+ Memo;
+ _ -> % If not, attempt to parse
+ Result = case ParseFun(Inp, StartIndex) of
+ {fail,_} = Failure -> % If it fails, memoize the failure
+ Failure;
+ {Match, InpRem, NewIndex} -> % If it passes, transform and memoize the result.
+ Transformed = TransformFun(Match, StartIndex),
+ {Transformed, InpRem, NewIndex}
+ end,
+ memoize(StartIndex, Name, Result),
+ Result
+ end.
+
+-spec setup_memo() -> ets:tid().
+setup_memo() ->
+ put({parse_memo_table, ?MODULE}, ets:new(?MODULE, [set])).
+
+-spec release_memo() -> true.
+release_memo() ->
+ ets:delete(memo_table_name()).
+
+-spec memoize(index(), atom(), parse_result()) -> true.
+memoize(Index, Name, Result) ->
+ Memo = case ets:lookup(memo_table_name(), Index) of
+ [] -> [];
+ [{Index, Plist}] -> Plist
+ end,
+ ets:insert(memo_table_name(), {Index, [{Name, Result}|Memo]}).
+
+-spec get_memo(index(), atom()) -> {ok, term()} | {error, not_found}.
+get_memo(Index, Name) ->
+ case ets:lookup(memo_table_name(), Index) of
+ [] -> {error, not_found};
+ [{Index, Plist}] ->
+ case proplists:lookup(Name, Plist) of
+ {Name, Result} -> {ok, Result};
+ _ -> {error, not_found}
+ end
+ end.
+
+-spec memo_table_name() -> ets:tid().
+memo_table_name() ->
+ get({parse_memo_table, ?MODULE}).
+
+-ifdef(p_eof).
+-spec p_eof() -> parse_fun().
+p_eof() ->
+ fun(<<>>, Index) -> {eof, [], Index};
+ (_, Index) -> {fail, {expected, eof, Index}} end.
+-endif.
+
+-ifdef(p_optional).
+-spec p_optional(parse_fun()) -> parse_fun().
+p_optional(P) ->
+ fun(Input, Index) ->
+ case P(Input, Index) of
+ {fail,_} -> {[], Input, Index};
+ {_, _, _} = Success -> Success
+ end
+ end.
+-endif.
+
+-ifdef(p_not).
+-spec p_not(parse_fun()) -> parse_fun().
+p_not(P) ->
+ fun(Input, Index)->
+ case P(Input,Index) of
+ {fail,_} ->
+ {[], Input, Index};
+ {Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_assert).
+-spec p_assert(parse_fun()) -> parse_fun().
+p_assert(P) ->
+ fun(Input,Index) ->
+ case P(Input,Index) of
+ {fail,_} = Failure-> Failure;
+ _ -> {[], Input, Index}
+ end
+ end.
+-endif.
+
+-ifdef(p_seq).
+-spec p_seq([parse_fun()]) -> parse_fun().
+p_seq(P) ->
+ fun(Input, Index) ->
+ p_all(P, Input, Index, [])
+ end.
+
+-spec p_all([parse_fun()], input(), index(), [term()]) -> parse_result().
+p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
+p_all([P|Parsers], Inp, Index, Accum) ->
+ case P(Inp, Index) of
+ {fail, _} = Failure -> Failure;
+ {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
+ end.
+-endif.
+
+-ifdef(p_choose).
+-spec p_choose([parse_fun()]) -> parse_fun().
+p_choose(Parsers) ->
+ fun(Input, Index) ->
+ p_attempt(Parsers, Input, Index, none)
+ end.
+
+-spec p_attempt([parse_fun()], input(), index(), none | parse_failure()) -> parse_result().
+p_attempt([], _Input, _Index, Failure) -> Failure;
+p_attempt([P|Parsers], Input, Index, FirstFailure)->
+ case P(Input, Index) of
+ {fail, _} = Failure ->
+ case FirstFailure of
+ none -> p_attempt(Parsers, Input, Index, Failure);
+ _ -> p_attempt(Parsers, Input, Index, FirstFailure)
+ end;
+ Result -> Result
+ end.
+-endif.
+
+-ifdef(p_zero_or_more).
+-spec p_zero_or_more(parse_fun()) -> parse_fun().
+p_zero_or_more(P) ->
+ fun(Input, Index) ->
+ p_scan(P, Input, Index, [])
+ end.
+-endif.
+
+-ifdef(p_one_or_more).
+-spec p_one_or_more(parse_fun()) -> parse_fun().
+p_one_or_more(P) ->
+ fun(Input, Index)->
+ Result = p_scan(P, Input, Index, []),
+ case Result of
+ {[_|_], _, _} ->
+ Result;
+ _ ->
+ {fail, {expected, Failure, _}} = P(Input,Index),
+ {fail, {expected, {at_least_one, Failure}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_label).
+-spec p_label(atom(), parse_fun()) -> parse_fun().
+p_label(Tag, P) ->
+ fun(Input, Index) ->
+ case P(Input, Index) of
+ {fail,_} = Failure ->
+ Failure;
+ {Result, InpRem, NewIndex} ->
+ {{Tag, Result}, InpRem, NewIndex}
+ end
+ end.
+-endif.
+
+-ifdef(p_scan).
+-spec p_scan(parse_fun(), input(), index(), [term()]) -> {[term()], input(), index()}.
+p_scan(_, <<>>, Index, Accum) -> {lists:reverse(Accum), <<>>, Index};
+p_scan(P, Inp, Index, Accum) ->
+ case P(Inp, Index) of
+ {fail,_} -> {lists:reverse(Accum), Inp, Index};
+ {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
+ end.
+-endif.
+
+-ifdef(p_string).
+-spec p_string(binary()) -> parse_fun().
+p_string(S) ->
+ Length = erlang:byte_size(S),
+ fun(Input, Index) ->
+ try
+ <<S:Length/binary, Rest/binary>> = Input,
+ {S, Rest, p_advance_index(S, Index)}
+ catch
+ error:{badmatch,_} -> {fail, {expected, {string, S}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_anything).
+-spec p_anything() -> parse_fun().
+p_anything() ->
+ fun(<<>>, Index) -> {fail, {expected, any_character, Index}};
+ (Input, Index) when is_binary(Input) ->
+ <<C/utf8, Rest/binary>> = Input,
+ {<<C/utf8>>, Rest, p_advance_index(<<C/utf8>>, Index)}
+ end.
+-endif.
+
+-ifdef(p_charclass).
+-spec p_charclass(string() | binary()) -> parse_fun().
+p_charclass(Class) ->
+ {ok, RE} = re:compile(Class, [unicode, dotall]),
+ fun(Inp, Index) ->
+ case re:run(Inp, RE, [anchored]) of
+ {match, [{0, Length}|_]} ->
+ {Head, Tail} = erlang:split_binary(Inp, Length),
+ {Head, Tail, p_advance_index(Head, Index)};
+ _ -> {fail, {expected, {character_class, binary_to_list(Class)}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_regexp).
+-spec p_regexp(binary()) -> parse_fun().
+p_regexp(Regexp) ->
+ {ok, RE} = re:compile(Regexp, [unicode, dotall, anchored]),
+ fun(Inp, Index) ->
+ case re:run(Inp, RE) of
+ {match, [{0, Length}|_]} ->
+ {Head, Tail} = erlang:split_binary(Inp, Length),
+ {Head, Tail, p_advance_index(Head, Index)};
+ _ -> {fail, {expected, {regexp, binary_to_list(Regexp)}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(line).
+-spec line(index() | term()) -> pos_integer() | undefined.
+line({{line,L},_}) -> L;
+line(_) -> undefined.
+-endif.
+
+-ifdef(column).
+-spec column(index() | term()) -> pos_integer() | undefined.
+column({_,{column,C}}) -> C;
+column(_) -> undefined.
+-endif.
+
+-spec p_advance_index(input() | unicode:charlist() | pos_integer(), index()) -> index().
+p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings
+ lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput));
+p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
+ {{line, Line}, {column, Col}} = Index,
+ case MatchedInput of
+ $\n -> {{line, Line+1}, {column, 1}};
+ _ -> {{line, Line}, {column, Col+1}}
+ end.
diff --git a/inttest/neotoma1/mock/neotoma/src/neotoma.app.src b/inttest/neotoma1/mock/neotoma/src/neotoma.app.src
new file mode 100644
index 0000000..dff6cef
--- /dev/null
+++ b/inttest/neotoma1/mock/neotoma/src/neotoma.app.src
@@ -0,0 +1,10 @@
+{application, neotoma,
+ [
+ {description, "PEG/Packrat toolkit and parser-generator."},
+ {vsn, "1.7.3"},
+ {applications, [kernel, stdlib]},
+ {contributors, ["Sean Cribbs"]},
+ {licenses, ["MIT"]},
+ {links, [{"Github", "https://github.com/seancribbs/neotoma"}]}
+ ]
+}.
diff --git a/inttest/neotoma1/mock/neotoma/src/neotoma.erl b/inttest/neotoma1/mock/neotoma/src/neotoma.erl
new file mode 100644
index 0000000..f4102f6
--- /dev/null
+++ b/inttest/neotoma1/mock/neotoma/src/neotoma.erl
@@ -0,0 +1,145 @@
+-module(neotoma).
+-author("Sean Cribbs <seancribbs@gmail.com>").
+-export([file/1, file/2, bootstrap/0]).
+-export([main/1]).
+
+-define(ALL_COMBINATORS, [p_eof, p_optional, p_not, p_assert, p_seq,
+ p_choose, p_zero_or_more, p_one_or_more, p_label, p_scan,
+ p_string, p_anything, p_charclass, p_regexp, line, column]).
+
+-type option() :: {module, atom()} | {output, file:filename()} | {transform_module, atom()} |
+ {neotoma_priv_dir, file:filename()}.
+
+%% @doc Handler function for escript.
+-spec main(list()) -> ok | no_return().
+main([]) ->
+ io:format("Usage: neotoma filename [-module output_module] [-output output_dir] [-transform_module transform_module]\n");
+main([Filename | Args]) ->
+ %% code:priv_dir is unreliable when called in escript context, but
+ %% escript:script_name does what we want.
+ PrivDir = filename:join([filename:dirname(escript:script_name()), "priv"]),
+ file(Filename, [{neotoma_priv_dir, PrivDir} | parse_options(Args)]).
+
+%% @doc Generates a parser from the specified file.
+%% @equiv file(Filename, [])
+-spec file(file:filename()) -> ok | {error, atom()}.
+file(InputGrammar) ->
+ file(InputGrammar, []).
+
+%% @doc Generates a parser from the specified file with the given options.
+-spec file(file:filename(), [option()]) -> ok | {error, atom()}.
+file(InputGrammar, Options) ->
+ Basename = filename:basename(InputGrammar, ".peg"),
+ InputDir = filename:dirname(InputGrammar),
+ ModuleName = proplists:get_value(module, Options, list_to_atom(Basename)),
+ OutputDir = proplists:get_value(output, Options, InputDir),
+ OutputFilename = filename:join(OutputDir, atom_to_list(ModuleName) ++ ".erl"),
+ TransformModule = proplists:get_value(transform_module, Options, false),
+ validate_params(filename:absname(InputGrammar),
+ ModuleName,
+ TransformModule,
+ filename:absname(OutputFilename)),
+ Parsed = parse_grammar(InputGrammar),
+ Rules = proplists:get_value(rules, Parsed),
+ Root = proplists:get_value(root, Parsed),
+ Code = proplists:get_value(code, Parsed),
+ GenTransform = proplists:get_value(transform, Parsed),
+ Combinators = proplists:get_value(combinators, Parsed, ?ALL_COMBINATORS),
+ ModuleAttrs = generate_module_attrs(ModuleName, Combinators),
+ EntryFuns = generate_entry_functions(Root),
+ TransformFun = create_transform(TransformModule, OutputDir, GenTransform),
+ PrivDir = proplists:get_value(neotoma_priv_dir, Options, code:priv_dir(neotoma)),
+ {ok, PegIncludes} = file:read_file(filename:join([PrivDir, "peg_includes.hrl"])),
+ file:write_file(OutputFilename, [ModuleAttrs, "\n", Code, "\n", EntryFuns, "\n", Rules, "\n", TransformFun, "\n", PegIncludes]).
+
+-spec validate_params(file:filename(),atom(),atom(),file:filename()) -> 'ok'.
+validate_params(InputGrammar, _, _, OutputFile) when InputGrammar =:= OutputFile ->
+ throw({badarg, "Input and output file are the same!"});
+validate_params(_,_, false, _) -> ok;
+validate_params(_,_, TransformModule, _) when not is_atom(TransformModule) ->
+ throw({badarg, "transform_module option must be an atom"});
+validate_params(_,Basename, TransformModule, _) when Basename =:= TransformModule ->
+ throw({badarg, "Transform module named same as parser module!"});
+validate_params(_,_, TransformModule, OutputFile) ->
+ OutMod = list_to_atom(filename:basename(OutputFile, ".erl")),
+ case OutMod of
+ TransformModule -> throw({badarg, "Transform module file same as parser output file!"});
+ _ -> ok
+ end.
+
+-spec generate_module_attrs(atom(), [atom()]) -> iolist().
+generate_module_attrs(ModName, Combinators) ->
+ ["-module(", atom_to_list(ModName) ,").\n",
+ "-export([parse/1,file/1]).\n",
+ [ generate_combinator_macro(C) || Combinators /= undefined, C <- Combinators ],
+ "\n"
+ ].
+
+generate_combinator_macro(C) ->
+ ["-define(", atom_to_list(C), ",true).\n"].
+
+-spec generate_entry_functions({iodata(),_}) -> iolist().
+generate_entry_functions(Root) ->
+ {RootRule,_} = Root,
+ ["-spec file(file:name()) -> any().\n",
+ "file(Filename) -> case file:read_file(Filename) of {ok,Bin} -> parse(Bin); Err -> Err end.\n\n",
+ "-spec parse(binary() | list()) -> any().\n",
+ "parse(List) when is_list(List) -> parse(unicode:characters_to_binary(List));\n",
+ "parse(Input) when is_binary(Input) ->\n",
+ " _ = setup_memo(),\n",
+ " Result = case '",RootRule,"'(Input,{{line,1},{column,1}}) of\n",
+ " {AST, <<>>, _Index} -> AST;\n",
+ " Any -> Any\n"
+ " end,\n",
+ " release_memo(), Result.\n"].
+
+-spec parse_grammar(file:filename()) -> any().
+parse_grammar(InputFile) ->
+ case neotoma_parse:file(InputFile) of
+ {fail, Index} ->
+ throw({grammar_error, {fail, Index}});
+ {Parsed, Remainder, Index} ->
+ io:format("WARNING: Grammar parse ended unexpectedly at ~p, generated parser may be incorrect.~nRemainder:~n~p",
+ [Index, Remainder]),
+ Parsed;
+ L when is_list(L) -> L;
+ _ -> throw({error, {unknown, grammar, InputFile}})
+ end.
+
+-spec create_transform(atom() | boolean(),file:filename(),_) -> iolist().
+create_transform(_,_,[]) -> [];
+create_transform(false,_,_) ->
+ "transform(_,Node,_Index) -> Node.";
+create_transform(ModName,Dir,_) when is_atom(ModName) ->
+ XfFile = filename:join(Dir, atom_to_list(ModName) ++ ".erl"),
+ case filelib:is_regular(XfFile) of
+ true -> io:format("'~s' already exists, skipping generation.~n", [XfFile]);
+ false -> generate_transform_stub(XfFile, ModName)
+ end,
+ ["transform(Symbol,Node,Index) -> ",atom_to_list(ModName),":transform(Symbol, Node, Index)."].
+
+-spec generate_transform_stub(file:filename(), atom()) -> 'ok' | {'error',atom()}.
+generate_transform_stub(XfFile,ModName) ->
+ Data = ["-module(",atom_to_list(ModName),").\n",
+ "-export([transform/3]).\n\n",
+ "%% Add clauses to this function to transform syntax nodes\n",
+ "%% from the parser into semantic output.\n",
+ "transform(Symbol, Node, _Index) when is_atom(Symbol) ->\n Node."],
+ file:write_file(XfFile, Data).
+
+%% @doc Bootstraps the neotoma metagrammar. Intended only for internal development!
+%% @equiv file("src/neotoma_parse.peg")
+-spec bootstrap() -> 'ok'.
+bootstrap() ->
+ file("priv/neotoma_parse.peg", [{output, "src/"}, {neotoma_priv_dir, "priv"}]).
+
+%% @doc Parses arguments passed to escript
+-spec parse_options(list()) -> list().
+parse_options(["-module", ModName | Rest]) ->
+ [{module, list_to_atom(ModName)} | parse_options(Rest)];
+parse_options(["-output", Dir | Rest]) ->
+ [{output, Dir} | parse_options(Rest)];
+parse_options(["-transform_module", ModName | Rest]) ->
+ [{transform_module, list_to_atom(ModName)} | parse_options(Rest)];
+parse_options([]) ->
+ [].
diff --git a/inttest/neotoma1/mock/neotoma/src/neotoma_parse.erl b/inttest/neotoma1/mock/neotoma/src/neotoma_parse.erl
new file mode 100644
index 0000000..1bffaf8
--- /dev/null
+++ b/inttest/neotoma1/mock/neotoma/src/neotoma_parse.erl
@@ -0,0 +1,625 @@
+-module(neotoma_parse).
+-export([parse/1,file/1]).
+-define(p_anything,true).
+-define(p_charclass,true).
+-define(p_choose,true).
+-define(p_label,true).
+-define(p_not,true).
+-define(p_one_or_more,true).
+-define(p_optional,true).
+-define(p_scan,true).
+-define(p_seq,true).
+-define(p_string,true).
+-define(p_zero_or_more,true).
+
+
+
+% insert escapes into a string
+-spec escape_string(string()) -> string().
+escape_string(String) -> escape_string(String, []).
+
+-spec escape_string(string(), string()) -> string().
+escape_string([], Output) ->
+ lists:reverse(Output);
+escape_string([H|T], Output) ->
+ escape_string(T,
+ case H of
+ $/ -> [$/,$\\|Output];
+ $\" -> [$\",$\\|Output]; % " comment inserted to help some editors with highlighting the generated parser
+ $\' -> [$\',$\\|Output]; % ' comment inserted to help some editors with highlighting the generated parser
+ $\b -> [$b,$\\|Output];
+ $\d -> [$d,$\\|Output];
+ $\e -> [$e,$\\|Output];
+ $\f -> [$f,$\\|Output];
+ $\n -> [$n,$\\|Output];
+ $\r -> [$r,$\\|Output];
+ $\s -> [$s,$\\|Output];
+ $\t -> [$t,$\\|Output];
+ $\v -> [$v,$\\|Output];
+ _ -> [H|Output]
+ end).
+
+-spec add_lhs(binary(), index()) -> true.
+add_lhs(Symbol, Index) ->
+ case ets:lookup(memo_table_name(), lhs) of
+ [] ->
+ ets:insert(memo_table_name(), {lhs, [{Symbol,Index}]});
+ [{lhs, L}] when is_list(L) ->
+ ets:insert(memo_table_name(), {lhs, [{Symbol,Index}|L]})
+ end.
+
+-spec add_nt(binary(), index()) -> true | ok.
+add_nt(Symbol, Index) ->
+ case ets:lookup(memo_table_name(), nts) of
+ [] ->
+ ets:insert(memo_table_name(), {nts, [{Symbol,Index}]});
+ [{nts, L}] when is_list(L) ->
+ case proplists:is_defined(Symbol, L) of
+ true ->
+ ok;
+ _ ->
+ ets:insert(memo_table_name(), {nts, [{Symbol,Index}|L]})
+ end
+ end.
+
+-spec verify_rules() -> ok | no_return().
+verify_rules() ->
+ [{lhs, LHS}] = ets:lookup(memo_table_name(), lhs),
+ [{nts, NTs}] = ets:lookup(memo_table_name(), nts),
+ [Root|NonRoots] = lists:reverse(LHS),
+ lists:foreach(fun({Sym,Idx}) ->
+ case proplists:is_defined(Sym, NTs) of
+ true ->
+ ok;
+ _ ->
+ io:format("neotoma warning: rule '~s' is unused. ~p~n", [Sym,Idx])
+ end
+ end, NonRoots),
+ lists:foreach(fun({S,I}) ->
+ case proplists:is_defined(S, LHS) of
+ true ->
+ ok;
+ _ ->
+ io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]),
+ exit({neotoma, {no_reduction, list_to_atom(binary_to_list(S))}})
+ end
+ end, NTs),
+ Root.
+
+-spec used_combinator(atom()) -> true.
+used_combinator(C) ->
+ case ets:lookup(memo_table_name(), combinators) of
+ [] ->
+ ets:insert(memo_table_name(), {combinators, ordsets:from_list([C])});
+ [{combinators, Cs}] ->
+ ets:insert(memo_table_name(), {combinators, ordsets:add_element(C, Cs)})
+ end.
+
+-spec used_transform_variables(binary()) -> [ 'Node' | 'Idx' ].
+used_transform_variables(Transform) ->
+ Code = unicode:characters_to_list(Transform),
+ {ok, Tokens, _} = erl_scan:string(Code),
+ used_transform_variables(Tokens, []).
+
+used_transform_variables([{var, _, Name}|Tokens], Acc) ->
+ used_transform_variables(Tokens, case Name of
+ 'Node' -> [Name | Acc];
+ 'Idx' -> [Name | Acc];
+ _ -> Acc
+ end);
+used_transform_variables([_|Tokens], Acc) ->
+ used_transform_variables(Tokens, Acc);
+used_transform_variables([], Acc) ->
+ lists:usort(Acc).
+
+-spec file(file:name()) -> any().
+file(Filename) -> case file:read_file(Filename) of {ok,Bin} -> parse(Bin); Err -> Err end.
+
+-spec parse(binary() | list()) -> any().
+parse(List) when is_list(List) -> parse(unicode:characters_to_binary(List));
+parse(Input) when is_binary(Input) ->
+ _ = setup_memo(),
+ Result = case 'rules'(Input,{{line,1},{column,1}}) of
+ {AST, <<>>, _Index} -> AST;
+ Any -> Any
+ end,
+ release_memo(), Result.
+
+-spec 'rules'(input(), index()) -> parse_result().
+'rules'(Input, Index) ->
+ p(Input, Index, 'rules', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), fun 'declaration_sequence'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, _Idx) ->
+ RootRule = verify_rules(),
+ Rules = unicode:characters_to_binary(lists:map(fun(R) -> [R, "\n\n"] end, lists:nth(2, Node))),
+ Code = case lists:nth(4, Node) of
+ {code, Block} -> Block;
+ _ -> []
+ end,
+ [{rules, Rules},
+ {code, Code},
+ {root, RootRule},
+ {transform, ets:lookup(memo_table_name(),gen_transform)},
+ {combinators, ets:lookup_element(memo_table_name(), combinators, 2)}]
+
+ end).
+
+-spec 'declaration_sequence'(input(), index()) -> parse_result().
+'declaration_sequence'(Input, Index) ->
+ p(Input, Index, 'declaration_sequence', fun(I,D) -> (p_seq([p_label('head', fun 'declaration'/2), p_label('tail', p_zero_or_more(p_seq([fun 'space'/2, fun 'declaration'/2])))]))(I,D) end, fun(Node, _Idx) ->
+ FirstRule = proplists:get_value(head, Node),
+ OtherRules = [I || [_,I] <- proplists:get_value(tail, Node, [])],
+ [FirstRule|OtherRules]
+ end).
+
+-spec 'declaration'(input(), index()) -> parse_result().
+'declaration'(Input, Index) ->
+ p(Input, Index, 'declaration', fun(I,D) -> (p_seq([fun 'nonterminal'/2, p_zero_or_more(fun 'space'/2), p_string(<<"<-">>), p_zero_or_more(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2), p_string(<<";">>)]))(I,D) end, fun(Node, _Idx) ->
+ [{nonterminal,Symbol}|Tail] = Node,
+ add_lhs(Symbol, Index),
+ Transform = case lists:nth(6,Tail) of
+ {code, CodeBlock} -> CodeBlock;
+ _ ->
+ ets:insert_new(memo_table_name(),{gen_transform, true}),
+ ["transform('",Symbol,"', Node, Idx)"]
+ end,
+ TransformArgs = case used_transform_variables(Transform) of
+ [] -> "_Node, _Idx";
+ ['Idx'] -> "_Node, Idx";
+ ['Node'] -> "Node, _Idx";
+ ['Idx', 'Node'] -> "Node, Idx"
+ end,
+ ["-spec '", Symbol, "'(input(), index()) -> parse_result().\n",
+ "'",Symbol,"'","(Input, Index) ->\n ",
+ "p(Input, Index, '",Symbol,"', fun(I,D) -> (",
+ lists:nth(4, Tail),
+ ")(I,D) end, fun(", TransformArgs, ") ->",Transform," end)."]
+ end).
+
+-spec 'parsing_expression'(input(), index()) -> parse_result().
+'parsing_expression'(Input, Index) ->
+ p(Input, Index, 'parsing_expression', fun(I,D) -> (p_choose([fun 'choice'/2, fun 'sequence'/2, fun 'primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'choice'(input(), index()) -> parse_result().
+'choice'(Input, Index) ->
+ p(Input, Index, 'choice', fun(I,D) -> (p_seq([p_label('head', fun 'alternative'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, p_string(<<"\/">>), fun 'space'/2, fun 'alternative'/2])))]))(I,D) end, fun(Node, _Idx) ->
+ Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)],
+ Head = proplists:get_value(head, Node),
+ Statements = [[", ", TS] || TS <- Tail],
+ used_combinator(p_choose),
+ ["p_choose([", Head, Statements, "])"]
+ end).
+
+-spec 'alternative'(input(), index()) -> parse_result().
+'alternative'(Input, Index) ->
+ p(Input, Index, 'alternative', fun(I,D) -> (p_choose([fun 'sequence'/2, fun 'labeled_primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'primary'(input(), index()) -> parse_result().
+'primary'(Input, Index) ->
+ p(Input, Index, 'primary', fun(I,D) -> (p_choose([p_seq([fun 'prefix'/2, fun 'atomic'/2]), p_seq([fun 'atomic'/2, fun 'suffix'/2]), fun 'atomic'/2]))(I,D) end, fun(Node, _Idx) ->
+case Node of
+ [Atomic, one_or_more] ->
+ used_combinator(p_one_or_more),
+ used_combinator(p_scan),
+ ["p_one_or_more(", Atomic, ")"];
+ [Atomic, zero_or_more] ->
+ used_combinator(p_zero_or_more),
+ used_combinator(p_scan),
+ ["p_zero_or_more(", Atomic, ")"];
+ [Atomic, optional] ->
+ used_combinator(p_optional),
+ ["p_optional(", Atomic, ")"];
+ [assert, Atomic] ->
+ used_combinator(p_assert),
+ ["p_assert(", Atomic, ")"];
+ [not_, Atomic] ->
+ used_combinator(p_not),
+ ["p_not(", Atomic, ")"];
+ _ -> Node
+end
+ end).
+
+-spec 'sequence'(input(), index()) -> parse_result().
+'sequence'(Input, Index) ->
+ p(Input, Index, 'sequence', fun(I,D) -> (p_seq([p_label('head', fun 'labeled_primary'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, fun 'labeled_primary'/2])))]))(I,D) end, fun(Node, _Idx) ->
+ Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)],
+ Head = proplists:get_value(head, Node),
+ Statements = [[", ", TS] || TS <- Tail],
+ used_combinator(p_seq),
+ ["p_seq([", Head, Statements, "])"]
+ end).
+
+-spec 'labeled_primary'(input(), index()) -> parse_result().
+'labeled_primary'(Input, Index) ->
+ p(Input, Index, 'labeled_primary', fun(I,D) -> (p_seq([p_optional(fun 'label'/2), fun 'primary'/2]))(I,D) end, fun(Node, _Idx) ->
+ case hd(Node) of
+ [] -> lists:nth(2, Node);
+ Label ->
+ used_combinator(p_label),
+ ["p_label('", Label, "', ", lists:nth(2, Node), ")"]
+ end
+ end).
+
+-spec 'label'(input(), index()) -> parse_result().
+'label'(Input, Index) ->
+ p(Input, Index, 'label', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2), p_string(<<":">>)]))(I,D) end, fun(Node, _Idx) ->
+ lists:sublist(Node, length(Node)-1)
+ end).
+
+-spec 'suffix'(input(), index()) -> parse_result().
+'suffix'(Input, Index) ->
+ p(Input, Index, 'suffix', fun(I,D) -> (p_choose([fun 'repetition_suffix'/2, fun 'optional_suffix'/2]))(I,D) end, fun(Node, _Idx) ->
+ case Node of
+ <<"*">> -> zero_or_more;
+ <<"+">> -> one_or_more;
+ <<"?">> -> optional
+ end
+ end).
+
+-spec 'optional_suffix'(input(), index()) -> parse_result().
+'optional_suffix'(Input, Index) ->
+ p(Input, Index, 'optional_suffix', fun(I,D) -> (p_string(<<"?">>))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'repetition_suffix'(input(), index()) -> parse_result().
+'repetition_suffix'(Input, Index) ->
+ p(Input, Index, 'repetition_suffix', fun(I,D) -> (p_choose([p_string(<<"+">>), p_string(<<"*">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'prefix'(input(), index()) -> parse_result().
+'prefix'(Input, Index) ->
+ p(Input, Index, 'prefix', fun(I,D) -> (p_choose([p_string(<<"&">>), p_string(<<"!">>)]))(I,D) end, fun(Node, _Idx) ->
+ case Node of
+ <<"&">> -> assert;
+ <<"!">> -> not_
+ end
+ end).
+
+-spec 'atomic'(input(), index()) -> parse_result().
+'atomic'(Input, Index) ->
+ p(Input, Index, 'atomic', fun(I,D) -> (p_choose([fun 'terminal'/2, fun 'nonterminal'/2, fun 'parenthesized_expression'/2]))(I,D) end, fun(Node, _Idx) ->
+case Node of
+ {nonterminal, Symbol} ->
+ [<<"fun '">>, Symbol, <<"'/2">>];
+ _ -> Node
+end
+ end).
+
+-spec 'parenthesized_expression'(input(), index()) -> parse_result().
+'parenthesized_expression'(Input, Index) ->
+ p(Input, Index, 'parenthesized_expression', fun(I,D) -> (p_seq([p_string(<<"(">>), p_optional(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_string(<<")">>)]))(I,D) end, fun(Node, _Idx) ->lists:nth(3, Node) end).
+
+-spec 'nonterminal'(input(), index()) -> parse_result().
+'nonterminal'(Input, Index) ->
+ p(Input, Index, 'nonterminal', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2)]))(I,D) end, fun(Node, Idx) ->
+ Symbol = unicode:characters_to_binary(Node),
+ add_nt(Symbol, Idx),
+ {nonterminal, Symbol}
+ end).
+
+-spec 'terminal'(input(), index()) -> parse_result().
+'terminal'(Input, Index) ->
+ p(Input, Index, 'terminal', fun(I,D) -> (p_choose([fun 'regexp_string'/2, fun 'quoted_string'/2, fun 'character_class'/2, fun 'anything_symbol'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'regexp_string'(input(), index()) -> parse_result().
+'regexp_string'(Input, Index) ->
+ p(Input, Index, 'regexp_string', fun(I,D) -> (p_seq([p_string(<<"#">>), p_label('string', p_one_or_more(p_seq([p_not(p_string(<<"#">>)), p_choose([p_string(<<"\\#">>), p_anything()])]))), p_string(<<"#">>)]))(I,D) end, fun(Node, _Idx) ->
+ used_combinator(p_regexp),
+ ["p_regexp(<<\"",
+ % Escape \ and " as they are used in erlang string. Other sumbol stay as is.
+ % \ -> \\
+ % " -> \"
+ re:replace(proplists:get_value(string, Node), "\"|\\\\", "\\\\&", [{return, binary}, global]),
+ "\">>)"]
+ end).
+
+-spec 'quoted_string'(input(), index()) -> parse_result().
+'quoted_string'(Input, Index) ->
+ p(Input, Index, 'quoted_string', fun(I,D) -> (p_choose([fun 'single_quoted_string'/2, fun 'double_quoted_string'/2]))(I,D) end, fun(Node, _Idx) ->
+ used_combinator(p_string),
+ lists:flatten(["p_string(<<\"",
+ escape_string(unicode:characters_to_list(proplists:get_value(string, Node))),
+ "\">>)"])
+ end).
+
+-spec 'double_quoted_string'(input(), index()) -> parse_result().
+'double_quoted_string'(Input, Index) ->
+ p(Input, Index, 'double_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\"">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\"">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\"">>), p_anything()])]))), p_string(<<"\"">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'single_quoted_string'(input(), index()) -> parse_result().
+'single_quoted_string'(Input, Index) ->
+ p(Input, Index, 'single_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\'">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\'">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\'">>), p_anything()])]))), p_string(<<"\'">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'character_class'(input(), index()) -> parse_result().
+'character_class'(Input, Index) ->
+ p(Input, Index, 'character_class', fun(I,D) -> (p_seq([p_string(<<"[">>), p_label('characters', p_one_or_more(p_seq([p_not(p_string(<<"]">>)), p_choose([p_seq([p_string(<<"\\\\">>), p_anything()]), p_seq([p_not(p_string(<<"\\\\">>)), p_anything()])])]))), p_string(<<"]">>)]))(I,D) end, fun(Node, _Idx) ->
+ used_combinator(p_charclass),
+ ["p_charclass(<<\"[",
+ escape_string(unicode:characters_to_list(proplists:get_value(characters, Node))),
+ "]\">>)"]
+ end).
+
+-spec 'anything_symbol'(input(), index()) -> parse_result().
+'anything_symbol'(Input, Index) ->
+ p(Input, Index, 'anything_symbol', fun(I,D) -> (p_string(<<".">>))(I,D) end, fun(_Node, _Idx) -> used_combinator(p_anything), <<"p_anything()">> end).
+
+-spec 'alpha_char'(input(), index()) -> parse_result().
+'alpha_char'(Input, Index) ->
+ p(Input, Index, 'alpha_char', fun(I,D) -> (p_charclass(<<"[A-Za-z_]">>))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'alphanumeric_char'(input(), index()) -> parse_result().
+'alphanumeric_char'(Input, Index) ->
+ p(Input, Index, 'alphanumeric_char', fun(I,D) -> (p_choose([fun 'alpha_char'/2, p_charclass(<<"[0-9]">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'space'(input(), index()) -> parse_result().
+'space'(Input, Index) ->
+ p(Input, Index, 'space', fun(I,D) -> (p_one_or_more(p_choose([fun 'white'/2, fun 'comment_to_eol'/2])))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'comment_to_eol'(input(), index()) -> parse_result().
+'comment_to_eol'(Input, Index) ->
+ p(Input, Index, 'comment_to_eol', fun(I,D) -> (p_seq([p_not(p_string(<<"%{">>)), p_string(<<"%">>), p_zero_or_more(p_seq([p_not(p_string(<<"\n">>)), p_anything()]))]))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'white'(input(), index()) -> parse_result().
+'white'(Input, Index) ->
+ p(Input, Index, 'white', fun(I,D) -> (p_charclass(<<"[\s\t\n\r]">>))(I,D) end, fun(Node, _Idx) ->Node end).
+
+-spec 'code_block'(input(), index()) -> parse_result().
+'code_block'(Input, Index) ->
+ p(Input, Index, 'code_block', fun(I,D) -> (p_choose([p_seq([p_string(<<"%{">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\%">>), p_string(<<"$%">>), p_seq([p_not(p_string(<<"%}">>)), p_anything()])]))), p_string(<<"%}">>)]), p_seq([p_string(<<"`">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\`">>), p_string(<<"$`">>), p_seq([p_not(p_string(<<"`">>)), p_anything()])]))), p_string(<<"`">>)]), p_string(<<"~">>)]))(I,D) end, fun(Node, _Idx) ->
+ case Node of
+ <<"~">> -> {code, <<"Node">>};
+ _ -> {code, proplists:get_value('code', Node)}
+ end
+ end).
+
+
+
+-file("peg_includes.hrl", 1).
+-type index() :: {{line, pos_integer()}, {column, pos_integer()}}.
+-type input() :: binary().
+-type parse_failure() :: {fail, term()}.
+-type parse_success() :: {term(), input(), index()}.
+-type parse_result() :: parse_failure() | parse_success().
+-type parse_fun() :: fun((input(), index()) -> parse_result()).
+-type xform_fun() :: fun((input(), index()) -> term()).
+
+-spec p(input(), index(), atom(), parse_fun(), xform_fun()) -> parse_result().
+p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
+ case get_memo(StartIndex, Name) of % See if the current reduction is memoized
+ {ok, Memo} -> %Memo; % If it is, return the stored result
+ Memo;
+ _ -> % If not, attempt to parse
+ Result = case ParseFun(Inp, StartIndex) of
+ {fail,_} = Failure -> % If it fails, memoize the failure
+ Failure;
+ {Match, InpRem, NewIndex} -> % If it passes, transform and memoize the result.
+ Transformed = TransformFun(Match, StartIndex),
+ {Transformed, InpRem, NewIndex}
+ end,
+ memoize(StartIndex, Name, Result),
+ Result
+ end.
+
+-spec setup_memo() -> ets:tid().
+setup_memo() ->
+ put({parse_memo_table, ?MODULE}, ets:new(?MODULE, [set])).
+
+-spec release_memo() -> true.
+release_memo() ->
+ ets:delete(memo_table_name()).
+
+-spec memoize(index(), atom(), parse_result()) -> true.
+memoize(Index, Name, Result) ->
+ Memo = case ets:lookup(memo_table_name(), Index) of
+ [] -> [];
+ [{Index, Plist}] -> Plist
+ end,
+ ets:insert(memo_table_name(), {Index, [{Name, Result}|Memo]}).
+
+-spec get_memo(index(), atom()) -> {ok, term()} | {error, not_found}.
+get_memo(Index, Name) ->
+ case ets:lookup(memo_table_name(), Index) of
+ [] -> {error, not_found};
+ [{Index, Plist}] ->
+ case proplists:lookup(Name, Plist) of
+ {Name, Result} -> {ok, Result};
+ _ -> {error, not_found}
+ end
+ end.
+
+-spec memo_table_name() -> ets:tid().
+memo_table_name() ->
+ get({parse_memo_table, ?MODULE}).
+
+-ifdef(p_eof).
+-spec p_eof() -> parse_fun().
+p_eof() ->
+ fun(<<>>, Index) -> {eof, [], Index};
+ (_, Index) -> {fail, {expected, eof, Index}} end.
+-endif.
+
+-ifdef(p_optional).
+-spec p_optional(parse_fun()) -> parse_fun().
+p_optional(P) ->
+ fun(Input, Index) ->
+ case P(Input, Index) of
+ {fail,_} -> {[], Input, Index};
+ {_, _, _} = Success -> Success
+ end
+ end.
+-endif.
+
+-ifdef(p_not).
+-spec p_not(parse_fun()) -> parse_fun().
+p_not(P) ->
+ fun(Input, Index)->
+ case P(Input,Index) of
+ {fail,_} ->
+ {[], Input, Index};
+ {Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_assert).
+-spec p_assert(parse_fun()) -> parse_fun().
+p_assert(P) ->
+ fun(Input,Index) ->
+ case P(Input,Index) of
+ {fail,_} = Failure-> Failure;
+ _ -> {[], Input, Index}
+ end
+ end.
+-endif.
+
+-ifdef(p_seq).
+-spec p_seq([parse_fun()]) -> parse_fun().
+p_seq(P) ->
+ fun(Input, Index) ->
+ p_all(P, Input, Index, [])
+ end.
+
+-spec p_all([parse_fun()], input(), index(), [term()]) -> parse_result().
+p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
+p_all([P|Parsers], Inp, Index, Accum) ->
+ case P(Inp, Index) of
+ {fail, _} = Failure -> Failure;
+ {Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
+ end.
+-endif.
+
+-ifdef(p_choose).
+-spec p_choose([parse_fun()]) -> parse_fun().
+p_choose(Parsers) ->
+ fun(Input, Index) ->
+ p_attempt(Parsers, Input, Index, none)
+ end.
+
+-spec p_attempt([parse_fun()], input(), index(), none | parse_failure()) -> parse_result().
+p_attempt([], _Input, _Index, Failure) -> Failure;
+p_attempt([P|Parsers], Input, Index, FirstFailure)->
+ case P(Input, Index) of
+ {fail, _} = Failure ->
+ case FirstFailure of
+ none -> p_attempt(Parsers, Input, Index, Failure);
+ _ -> p_attempt(Parsers, Input, Index, FirstFailure)
+ end;
+ Result -> Result
+ end.
+-endif.
+
+-ifdef(p_zero_or_more).
+-spec p_zero_or_more(parse_fun()) -> parse_fun().
+p_zero_or_more(P) ->
+ fun(Input, Index) ->
+ p_scan(P, Input, Index, [])
+ end.
+-endif.
+
+-ifdef(p_one_or_more).
+-spec p_one_or_more(parse_fun()) -> parse_fun().
+p_one_or_more(P) ->
+ fun(Input, Index)->
+ Result = p_scan(P, Input, Index, []),
+ case Result of
+ {[_|_], _, _} ->
+ Result;
+ _ ->
+ {fail, {expected, Failure, _}} = P(Input,Index),
+ {fail, {expected, {at_least_one, Failure}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_label).
+-spec p_label(atom(), parse_fun()) -> parse_fun().
+p_label(Tag, P) ->
+ fun(Input, Index) ->
+ case P(Input, Index) of
+ {fail,_} = Failure ->
+ Failure;
+ {Result, InpRem, NewIndex} ->
+ {{Tag, Result}, InpRem, NewIndex}
+ end
+ end.
+-endif.
+
+-ifdef(p_scan).
+-spec p_scan(parse_fun(), input(), index(), [term()]) -> {[term()], input(), index()}.
+p_scan(_, <<>>, Index, Accum) -> {lists:reverse(Accum), <<>>, Index};
+p_scan(P, Inp, Index, Accum) ->
+ case P(Inp, Index) of
+ {fail,_} -> {lists:reverse(Accum), Inp, Index};
+ {Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
+ end.
+-endif.
+
+-ifdef(p_string).
+-spec p_string(binary()) -> parse_fun().
+p_string(S) ->
+ Length = erlang:byte_size(S),
+ fun(Input, Index) ->
+ try
+ <<S:Length/binary, Rest/binary>> = Input,
+ {S, Rest, p_advance_index(S, Index)}
+ catch
+ error:{badmatch,_} -> {fail, {expected, {string, S}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_anything).
+-spec p_anything() -> parse_fun().
+p_anything() ->
+ fun(<<>>, Index) -> {fail, {expected, any_character, Index}};
+ (Input, Index) when is_binary(Input) ->
+ <<C/utf8, Rest/binary>> = Input,
+ {<<C/utf8>>, Rest, p_advance_index(<<C/utf8>>, Index)}
+ end.
+-endif.
+
+-ifdef(p_charclass).
+-spec p_charclass(string() | binary()) -> parse_fun().
+p_charclass(Class) ->
+ {ok, RE} = re:compile(Class, [unicode, dotall]),
+ fun(Inp, Index) ->
+ case re:run(Inp, RE, [anchored]) of
+ {match, [{0, Length}|_]} ->
+ {Head, Tail} = erlang:split_binary(Inp, Length),
+ {Head, Tail, p_advance_index(Head, Index)};
+ _ -> {fail, {expected, {character_class, binary_to_list(Class)}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(p_regexp).
+-spec p_regexp(binary()) -> parse_fun().
+p_regexp(Regexp) ->
+ {ok, RE} = re:compile(Regexp, [unicode, dotall, anchored]),
+ fun(Inp, Index) ->
+ case re:run(Inp, RE) of
+ {match, [{0, Length}|_]} ->
+ {Head, Tail} = erlang:split_binary(Inp, Length),
+ {Head, Tail, p_advance_index(Head, Index)};
+ _ -> {fail, {expected, {regexp, binary_to_list(Regexp)}, Index}}
+ end
+ end.
+-endif.
+
+-ifdef(line).
+-spec line(index() | term()) -> pos_integer() | undefined.
+line({{line,L},_}) -> L;
+line(_) -> undefined.
+-endif.
+
+-ifdef(column).
+-spec column(index() | term()) -> pos_integer() | undefined.
+column({_,{column,C}}) -> C;
+column(_) -> undefined.
+-endif.
+
+-spec p_advance_index(input() | unicode:charlist() | pos_integer(), index()) -> index().
+p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings
+ lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput));
+p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
+ {{line, Line}, {column, Col}} = Index,
+ case MatchedInput of
+ $\n -> {{line, Line+1}, {column, 1}};
+ _ -> {{line, Line}, {column, Col+1}}
+ end.
diff --git a/inttest/neotoma1/neotoma_src_rt.erl b/inttest/neotoma1/neotoma_src_rt.erl
new file mode 100644
index 0000000..6f7c6ff
--- /dev/null
+++ b/inttest/neotoma1/neotoma_src_rt.erl
@@ -0,0 +1,81 @@
+%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+%% -------------------------------------------------------------------
+%%
+%% rebar: Erlang Build Tools
+%%
+%% Copyright (c) 2015 Luis Rascao (luis.rascao@gmail.com)
+%%
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+%%
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+%% -------------------------------------------------------------------
+-module(neotoma_src_rt).
+
+-compile(export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("deps/retest/include/retest.hrl").
+
+-define(GENERATED_MODULES,
+ [csv_peg]).
+
+files() ->
+ [{copy, "../../rebar", "rebar"},
+ {copy, "rebar.config", "rebar.config"},
+ {copy, "mock", "deps"},
+ {copy, "src", "src"}].
+
+run(Dir) ->
+ retest_log:log(debug, "Running in Dir: ~s~n", [Dir]),
+ ?assertMatch({ok, _}, retest:sh("./rebar compile",
+ [{async, false}])),
+
+ ok = check_files_generated(),
+
+ retest_log:log(debug, "Verify cleanup~n", []),
+ ?assertMatch({ok, _}, retest_sh:run("./rebar clean",
+ [])),
+ ok = check_files_deleted(),
+ ok.
+
+check_files_generated() ->
+ check(fun filelib:is_regular/1,
+ generated_erl_files()).
+
+check_files_deleted() ->
+ check(fun file_does_not_exist/1,
+ generated_erl_files()).
+
+generated_erl_files() ->
+ add_dir("src", add_ext(?GENERATED_MODULES, ".erl")).
+
+add_ext(Modules, Ext) ->
+ [lists:concat([Module, Ext]) || Module <- Modules].
+
+add_dir(Dir, Files) ->
+ [filename:join(Dir, File) || File <- Files].
+
+file_does_not_exist(F) ->
+ not filelib:is_regular(F).
+
+check(Check, Files) ->
+ lists:foreach(
+ fun(F) ->
+ ?assertMatch({true, _}, {Check(F), F})
+ end,
+ Files).
diff --git a/inttest/neotoma1/rebar.config b/inttest/neotoma1/rebar.config
new file mode 100644
index 0000000..8757da8
--- /dev/null
+++ b/inttest/neotoma1/rebar.config
@@ -0,0 +1,15 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+
+{deps,
+ [
+ %% The dependency below to neotoma is needed for "rebar compile" to
+ %% work, thus for the inttest to work, but the neotoma that is actually
+ %% used in inttest is brought in from the inttest/neotoma1/mock
+ %% subdirectory.
+ {neotoma, ".*"}
+ ]}.
+
+{neotoma_opts, [
+ {module_ext, "_peg"}
+]}.
diff --git a/inttest/neotoma1/src/csv.peg b/inttest/neotoma1/src/csv.peg
new file mode 100644
index 0000000..000c00f
--- /dev/null
+++ b/inttest/neotoma1/src/csv.peg
@@ -0,0 +1,31 @@
+rows <- head:row tail:(crlf row)* / ''
+`
+case Node of
+ [] -> [];
+ [""] -> [];
+ _ ->
+ Head = proplists:get_value(head, Node),
+ Tail = [R || [_,R] <- proplists:get_value(tail, Node)],
+ [Head|Tail]
+end
+`;
+
+row <- head:field tail:(field_sep field)* / ''
+`
+case Node of
+ [] -> [];
+ [""] -> [];
+ _ ->
+ Head = proplists:get_value(head, Node),
+ Tail = [F || [_,F] <- proplists:get_value(tail, Node)],
+ [Head|Tail]
+end
+`;
+field <- quoted_field / (!field_sep !crlf .)* `iolist_to_binary(Node)`;
+quoted_field <- '"' string:('""' / (!'"' .))* '"'
+`
+ String = proplists:get_value(string, Node),
+ re:replace(String, "[\"]{2}", "\"",[global, {return, binary}])
+`;
+field_sep <- ',' ~;
+crlf <- [\r]? [\n] ~;
diff --git a/inttest/neotoma1/src/neotoma1.app.src b/inttest/neotoma1/src/neotoma1.app.src
new file mode 100644
index 0000000..96a50e5
--- /dev/null
+++ b/inttest/neotoma1/src/neotoma1.app.src
@@ -0,0 +1,6 @@
+{application, neotoma1,
+ [{vsn, "1"},
+ {modules, []},
+ {registered, []},
+ {applications, [kernel, stdlib]}]}.
+
