Diginet's code

% lexer.pro
:- use_module(library(pure_input)).

phrase_test(Grammar,Str) :- string_codes(Str,Codes),phrase(Grammar,Codes).

tokenize(File,Z) :-
 setup_call_cleanup(
   open(File,read,Stream),
   stream_to_lazy_list(Stream,List),
   phrase(tokens(Z),List),
   close(Stream)),!.


string([H|T]) --> {code_type(H,alpha)},[H],(string(T); ([],{T=[]})).
number([H|T]) --> {code_type(H,digit)},[H],(number(T); ([],{T=[]})).

left_quote([_|T]) --> `\``,(left_quote(T); ([],{T=[]})).
right_quote([_|T]) --> `'`,(right_quote(T); ([],{T=[]})).

reserved("Program").
reserved("Use").
reserved("Const").
reserved("Type").
reserved("Var").
reserved("Code").
reserved("End").


% Strip spaces, tabs and newlines.
tokens([]) --> [C],{var(C)},!.
tokens(HT) --> [C], {code_type(C,white)},!,tokens(HT).
tokens([H|T]) --> tokenZ(H),tokens(T).
% Anything not matched counted as error
tokens(syntax_err(X,Line,LinePos)) --> [C],lazy_list_location(file(_, Line, LinePos,_)), {name(X, [C])}.
tokens([]) --> [].

tokenZ(Make)--> did_token(Match,Make),Match,!.
tokenZ(integer(N)) --> number(W), {number_codes(N,W)}.
% Delimiters
% tokenZ(quote("left",N)) --> left_quote(W), {length(W,N)}.
% tokenZ(quote("right",N),Str) -->  right_quote(W), {length(W,N)}.
tokenZ(Z) -->
   lazy_list_location(file(_, Line, LinePos,_)),
  string(W),
   {string_codes(S,W),
    (reserved(S)->
     Z = keyword(S,Line:LinePos) ;
     Z = ident(S,Line:LinePos))}.





did_token(`{`,delimiter("brace","left")).
did_token(`}`,delimiter("brace","right")).
did_token(`(`,delimiter("paren","left")).
did_token(`)`,delimiter("paren","right")).


% Structural
did_token(`:`,punct("colon")).
did_token(`;`,punct("semicolon")).
did_token(`,`,punct("comma")).
did_token(`#`,punct("hash")).

% declare new type constructors, using previously designed types
% OCaml paramterized modules

% Assignment Operator
did_token(`:=`,assignment).
did_token(`<=`,param_assignment).

% Comparison Operators
did_token(`==`,equal).
did_token(`=/`,not_equal).
did_token(`>=`,greater_equal).
did_token(`=<`,less_equal).
did_token(`>`,greater_than).
did_token(`<`,less_than).

did_token(`=`,equal_sign).
did_token(`.`,dot).

% Arithmetic Operators
did_token(`+`,operator("add")).
did_token(`-`,operator("subtract")).
did_token(`*`,operator("multiply")).
did_token(`/`,operator("divide")).
did_token(`^`,operator("power")).