727 lines
35 KiB
Plaintext
727 lines
35 KiB
Plaintext
implementation module Lexer;
|
|
|
|
from FIO import ReadNBytes;
|
|
from SYSTEM import ADR;
|
|
|
|
from Storage import DEALLOCATE, ALLOCATE;
|
|
from Strings import Length;
|
|
from MemUtils import MemZero;
|
|
from StrCase import Lower;
|
|
|
|
const
|
|
ChunkSize = 65536;
|
|
|
|
type
|
|
(*
|
|
* Classification table assigns each possible character to a group (class). All
|
|
* characters of the same group a handled equivalently.
|
|
*
|
|
* Classification:
|
|
*)
|
|
TransitionClass = (
|
|
transitionClassInvalid,
|
|
transitionClassDigit,
|
|
transitionClassAlpha,
|
|
transitionClassSpace,
|
|
transitionClassColon,
|
|
transitionClassEquals,
|
|
transitionClassLeftParen,
|
|
transitionClassRightParen,
|
|
transitionClassAsterisk,
|
|
transitionClassUnderscore,
|
|
transitionClassSingle,
|
|
transitionClassHex,
|
|
transitionClassZero,
|
|
transitionClassX,
|
|
transitionClassEof,
|
|
transitionClassDot,
|
|
transitionClassMinus,
|
|
transitionClassSingleQuote,
|
|
transitionClassDoubleQuote,
|
|
transitionClassGreater,
|
|
transitionClassLess,
|
|
transitionClassOther
|
|
);
|
|
TransitionState = (
|
|
transitionStateStart,
|
|
transitionStateColon,
|
|
transitionStateIdentifier,
|
|
transitionStateDecimal,
|
|
transitionStateGreater,
|
|
transitionStateMinus,
|
|
transitionStateLeftParen,
|
|
transitionStateLess,
|
|
transitionStateDot,
|
|
transitionStateComment,
|
|
transitionStateClosingComment,
|
|
transitionStateCharacter,
|
|
transitionStateString,
|
|
transitionStateLeadingZero,
|
|
transitionStateDecimalSuffix,
|
|
transitionStateEnd
|
|
);
|
|
TransitionAction = procedure(PLexer, PLexerToken);
|
|
Transition = record
|
|
Action: TransitionAction;
|
|
NextState: TransitionState
|
|
end;
|
|
|
|
VAR
|
|
Classification: ARRAY[1..128] OF TransitionClass;
|
|
Transitions: ARRAY[0..MAX(TransitionState)] OF ARRAY[0..MAX(TransitionClass)] OF Transition;
|
|
|
|
PROCEDURE InitializeClassification();
|
|
BEGIN
|
|
Classification[1] := transitionClassEof; (* NUL *)
|
|
Classification[2] := transitionClassInvalid; (* SOH *)
|
|
Classification[3] := transitionClassInvalid; (* STX *)
|
|
Classification[4] := transitionClassInvalid; (* ETX *)
|
|
Classification[5] := transitionClassInvalid; (* EOT *)
|
|
Classification[6] := transitionClassInvalid; (* EMQ *)
|
|
Classification[7] := transitionClassInvalid; (* ACK *)
|
|
Classification[8] := transitionClassInvalid; (* BEL *)
|
|
Classification[9] := transitionClassInvalid; (* BS *)
|
|
Classification[10] := transitionClassSpace; (* HT *)
|
|
Classification[11] := transitionClassSpace; (* LF *)
|
|
Classification[12] := transitionClassInvalid; (* VT *)
|
|
Classification[13] := transitionClassInvalid; (* FF *)
|
|
Classification[14] := transitionClassSpace; (* CR *)
|
|
Classification[15] := transitionClassInvalid; (* SO *)
|
|
Classification[16] := transitionClassInvalid; (* SI *)
|
|
Classification[17] := transitionClassInvalid; (* DLE *)
|
|
Classification[18] := transitionClassInvalid; (* DC1 *)
|
|
Classification[19] := transitionClassInvalid; (* DC2 *)
|
|
Classification[20] := transitionClassInvalid; (* DC3 *)
|
|
Classification[21] := transitionClassInvalid; (* DC4 *)
|
|
Classification[22] := transitionClassInvalid; (* NAK *)
|
|
Classification[23] := transitionClassInvalid; (* SYN *)
|
|
Classification[24] := transitionClassInvalid; (* ETB *)
|
|
Classification[25] := transitionClassInvalid; (* CAN *)
|
|
Classification[26] := transitionClassInvalid; (* EM *)
|
|
Classification[27] := transitionClassInvalid; (* SUB *)
|
|
Classification[28] := transitionClassInvalid; (* ESC *)
|
|
Classification[29] := transitionClassInvalid; (* FS *)
|
|
Classification[30] := transitionClassInvalid; (* GS *)
|
|
Classification[31] := transitionClassInvalid; (* RS *)
|
|
Classification[32] := transitionClassInvalid; (* US *)
|
|
Classification[33] := transitionClassSpace; (* Space *)
|
|
Classification[34] := transitionClassSingle; (* ! *)
|
|
Classification[35] := transitionClassDoubleQuote; (* " *)
|
|
Classification[36] := transitionClassOther; (* # *)
|
|
Classification[37] := transitionClassOther; (* $ *)
|
|
Classification[38] := transitionClassSingle; (* % *)
|
|
Classification[39] := transitionClassSingle; (* & *)
|
|
Classification[40] := transitionClassSingleQuote; (* ' *)
|
|
Classification[41] := transitionClassLeftParen; (* ( *)
|
|
Classification[42] := transitionClassRightParen; (* ) *)
|
|
Classification[43] := transitionClassAsterisk; (* * *)
|
|
Classification[44] := transitionClassSingle; (* + *)
|
|
Classification[45] := transitionClassSingle; (* , *)
|
|
Classification[46] := transitionClassMinus; (* - *)
|
|
Classification[47] := transitionClassDot; (* . *)
|
|
Classification[48] := transitionClassSingle; (* / *)
|
|
Classification[49] := transitionClassZero; (* 0 *)
|
|
Classification[50] := transitionClassDigit; (* 1 *)
|
|
Classification[51] := transitionClassDigit; (* 2 *)
|
|
Classification[52] := transitionClassDigit; (* 3 *)
|
|
Classification[53] := transitionClassDigit; (* 4 *)
|
|
Classification[54] := transitionClassDigit; (* 5 *)
|
|
Classification[55] := transitionClassDigit; (* 6 *)
|
|
Classification[56] := transitionClassDigit; (* 7 *)
|
|
Classification[57] := transitionClassDigit; (* 8 *)
|
|
Classification[58] := transitionClassDigit; (* 9 *)
|
|
Classification[59] := transitionClassColon; (* : *)
|
|
Classification[60] := transitionClassSingle; (* ; *)
|
|
Classification[61] := transitionClassLess; (* < *)
|
|
Classification[62] := transitionClassEquals; (* = *)
|
|
Classification[63] := transitionClassGreater; (* > *)
|
|
Classification[64] := transitionClassOther; (* ? *)
|
|
Classification[65] := transitionClassSingle; (* @ *)
|
|
Classification[66] := transitionClassAlpha; (* A *)
|
|
Classification[67] := transitionClassAlpha; (* B *)
|
|
Classification[68] := transitionClassAlpha; (* C *)
|
|
Classification[69] := transitionClassAlpha; (* D *)
|
|
Classification[70] := transitionClassAlpha; (* E *)
|
|
Classification[71] := transitionClassAlpha; (* F *)
|
|
Classification[72] := transitionClassAlpha; (* G *)
|
|
Classification[73] := transitionClassAlpha; (* H *)
|
|
Classification[74] := transitionClassAlpha; (* I *)
|
|
Classification[75] := transitionClassAlpha; (* J *)
|
|
Classification[76] := transitionClassAlpha; (* K *)
|
|
Classification[77] := transitionClassAlpha; (* L *)
|
|
Classification[78] := transitionClassAlpha; (* M *)
|
|
Classification[79] := transitionClassAlpha; (* N *)
|
|
Classification[80] := transitionClassAlpha; (* O *)
|
|
Classification[81] := transitionClassAlpha; (* P *)
|
|
Classification[82] := transitionClassAlpha; (* Q *)
|
|
Classification[83] := transitionClassAlpha; (* R *)
|
|
Classification[84] := transitionClassAlpha; (* S *)
|
|
Classification[85] := transitionClassAlpha; (* T *)
|
|
Classification[86] := transitionClassAlpha; (* U *)
|
|
Classification[87] := transitionClassAlpha; (* V *)
|
|
Classification[88] := transitionClassAlpha; (* W *)
|
|
Classification[89] := transitionClassAlpha; (* X *)
|
|
Classification[90] := transitionClassAlpha; (* Y *)
|
|
Classification[91] := transitionClassAlpha; (* Z *)
|
|
Classification[92] := transitionClassSingle; (* [ *)
|
|
Classification[93] := transitionClassOther; (* \ *)
|
|
Classification[94] := transitionClassSingle; (* ] *)
|
|
Classification[95] := transitionClassSingle; (* ^ *)
|
|
Classification[96] := transitionClassUnderscore; (* _ *)
|
|
Classification[97] := transitionClassOther; (* ` *)
|
|
Classification[98] := transitionClassHex; (* a *)
|
|
Classification[99] := transitionClassHex; (* b *)
|
|
Classification[100] := transitionClassHex; (* c *)
|
|
Classification[101] := transitionClassHex; (* d *)
|
|
Classification[102] := transitionClassHex; (* e *)
|
|
Classification[103] := transitionClassHex; (* f *)
|
|
Classification[104] := transitionClassAlpha; (* g *)
|
|
Classification[105] := transitionClassAlpha; (* h *)
|
|
Classification[106] := transitionClassAlpha; (* i *)
|
|
Classification[107] := transitionClassAlpha; (* j *)
|
|
Classification[108] := transitionClassAlpha; (* k *)
|
|
Classification[109] := transitionClassAlpha; (* l *)
|
|
Classification[110] := transitionClassAlpha; (* m *)
|
|
Classification[111] := transitionClassAlpha; (* n *)
|
|
Classification[112] := transitionClassAlpha; (* o *)
|
|
Classification[113] := transitionClassAlpha; (* p *)
|
|
Classification[114] := transitionClassAlpha; (* q *)
|
|
Classification[115] := transitionClassAlpha; (* r *)
|
|
Classification[116] := transitionClassAlpha; (* s *)
|
|
Classification[117] := transitionClassAlpha; (* t *)
|
|
Classification[118] := transitionClassAlpha; (* u *)
|
|
Classification[119] := transitionClassAlpha; (* v *)
|
|
Classification[120] := transitionClassAlpha; (* w *)
|
|
Classification[121] := transitionClassX; (* x *)
|
|
Classification[122] := transitionClassAlpha; (* y *)
|
|
Classification[123] := transitionClassAlpha; (* z *)
|
|
Classification[124] := transitionClassOther; (* { *)
|
|
Classification[125] := transitionClassSingle; (* | *)
|
|
Classification[126] := transitionClassOther; (* } *)
|
|
Classification[127] := transitionClassSingle; (* ~ *)
|
|
Classification[128] := transitionClassInvalid (* DEL *)
|
|
END InitializeClassification;
|
|
|
|
PROCEDURE CompareKeyword(Keyword: ARRAY OF CHAR; TokenStart: PLexerBuffer; TokenEnd: PLexerBuffer): BOOLEAN;
|
|
VAR
|
|
Result: BOOLEAN;
|
|
Index: CARDINAL;
|
|
BEGIN
|
|
Index := 0;
|
|
Result := TRUE;
|
|
|
|
WHILE (Index < Length(Keyword)) AND (TokenStart <> TokenEnd) AND Result DO
|
|
Result := (Keyword[Index] = TokenStart^) OR (Lower(Keyword[Index]) = TokenStart^);
|
|
INC(TokenStart);
|
|
INC(Index)
|
|
END;
|
|
RETURN (Index = Length(Keyword)) AND (TokenStart = TokenEnd) AND Result
|
|
END CompareKeyword;
|
|
|
|
(* Reached the end of file. *)
|
|
PROCEDURE TransitionActionEof(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
AToken^.Kind := lexerKindEof
|
|
END TransitionActionEof;
|
|
|
|
(* Add the character to the token currently read and advance to the next character. *)
|
|
PROCEDURE TransitionActionAccumulate(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
INC(ALexer^.Current)
|
|
END TransitionActionAccumulate;
|
|
|
|
(* The current character is not a part of the token. Finish the token already
|
|
* read. Don't advance to the next character. *)
|
|
PROCEDURE TransitionActionFinalize(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
IF ALexer^.Start^ = ':' THEN
|
|
AToken^.Kind := lexerKindColon
|
|
ELSIF ALexer^.Start^ = '>' THEN
|
|
AToken^.Kind := lexerKindGreaterThan
|
|
ELSIF ALexer^.Start^ = '<' THEN
|
|
AToken^.Kind := lexerKindLessThan
|
|
ELSIF ALexer^.Start^ = '(' THEN
|
|
AToken^.Kind := lexerKindLeftParen
|
|
ELSIF ALexer^.Start^ = '-' THEN
|
|
AToken^.Kind := lexerKindLeftParen
|
|
ELSIF ALexer^.Start^ = '.' THEN
|
|
AToken^.Kind := lexerKindDot
|
|
END
|
|
END TransitionActionFinalize;
|
|
|
|
(* An action for tokens containing multiple characters. *)
|
|
PROCEDURE TransitionActionComposite(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
IF ALexer^.Start^ = '<' THEN
|
|
IF ALexer^.Current^ = '>' THEN
|
|
AToken^.Kind := lexerKindNotEqual
|
|
ELSIF ALexer^.Current^ = '=' THEN
|
|
AToken^.Kind := lexerKindLessEqual
|
|
END
|
|
ELSIF (ALexer^.Start^ = '>') AND (ALexer^.Current^ = '=') THEN
|
|
AToken^.Kind := lexerKindGreaterEqual
|
|
ELSIF (ALexer^.Start^ = '.') AND (ALexer^.Current^ = '.') THEN
|
|
AToken^.Kind := lexerKindRange
|
|
ELSIF (ALexer^.Start^ = ':') AND (ALexer^.Current^ = '=') THEN
|
|
AToken^.Kind := lexerKindAssignment
|
|
END;
|
|
INC(ALexer^.Current)
|
|
END TransitionActionComposite;
|
|
|
|
(* Skip a space. *)
|
|
PROCEDURE TransitionActionSkip(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
INC(ALexer^.Current);
|
|
INC(ALexer^.Start)
|
|
END TransitionActionSkip;
|
|
|
|
(* 0x04. Delimited string action. *)
|
|
PROCEDURE TransitionActionDelimited(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
IF ALexer^.Start^ = '(' THEN
|
|
AToken^.Kind := lexerKindComment
|
|
ELSIF ALexer^.Start^ = '"' THEN
|
|
AToken^.Kind := lexerKindCharacter
|
|
ELSIF ALexer^.Start^ = "'" THEN
|
|
AToken^.Kind := lexerKindString
|
|
END;
|
|
INC(ALexer^.Current)
|
|
END TransitionActionDelimited;
|
|
|
|
(* Finalize keyword or identifier. *)
|
|
PROCEDURE TransitionActionKeyId(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
IF CompareKeyword('PROGRAM', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindProgram
|
|
ELSIF CompareKeyword('IMPORT', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindImport
|
|
ELSIF CompareKeyword('CONST', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindConst
|
|
ELSIF CompareKeyword('VAR', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindVar
|
|
ELSIF CompareKeyword('IF', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindIf
|
|
ELSIF CompareKeyword('THEN', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindThen
|
|
ELSIF CompareKeyword('ELSIF', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindElsif
|
|
ELSIF CompareKeyword('ELSE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindElse
|
|
ELSIF CompareKeyword('WHILE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindWhile
|
|
ELSIF CompareKeyword('DO', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindDo
|
|
ELSIF CompareKeyword('PROCEDURE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindProc
|
|
ELSIF CompareKeyword('BEGIN', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindBegin
|
|
ELSIF CompareKeyword('END', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindEnd
|
|
ELSIF CompareKeyword('TYPE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindType
|
|
ELSIF CompareKeyword('RECORD', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindRecord
|
|
ELSIF CompareKeyword('UNION', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindUnion
|
|
ELSIF CompareKeyword('NIL', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindNull
|
|
ELSIF CompareKeyword('AND', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindAnd
|
|
ELSIF CompareKeyword('OR', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindOr
|
|
ELSIF CompareKeyword('RETURN', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindReturn
|
|
ELSIF CompareKeyword('DEFINITION', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindDefinition
|
|
ELSIF CompareKeyword('TO', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindTo
|
|
ELSIF CompareKeyword('CASE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindCase
|
|
ELSIF CompareKeyword('OF', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindOf
|
|
ELSIF CompareKeyword('FROM', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindFrom
|
|
ELSIF CompareKeyword('MODULE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindModule
|
|
ELSIF CompareKeyword('IMPLEMENTATION', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindImplementation
|
|
ELSIF CompareKeyword('POINTER', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindPointer
|
|
ELSIF CompareKeyword('ARRAY', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindArray
|
|
ELSIF CompareKeyword('TRUE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindBoolean;
|
|
AToken^.booleanKind := TRUE
|
|
ELSIF CompareKeyword('FALSE', ALexer^.Start, ALexer^.Current) THEN
|
|
AToken^.Kind := lexerKindBoolean;
|
|
AToken^.booleanKind := FALSE
|
|
ELSE
|
|
AToken^.Kind := lexerKindIdentifier
|
|
END;
|
|
END TransitionActionKeyId;
|
|
|
|
(* Action for tokens containing only one character. The character cannot be
|
|
* followed by other characters forming a composite token. *)
|
|
PROCEDURE TransitionActionSingle(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
CASE ALexer^.Current^ OF
|
|
'&': AToken^.Kind := lexerKindAnd |
|
|
';': AToken^.Kind := lexerKindSemicolon |
|
|
',': AToken^.Kind := lexerKindComma |
|
|
')': AToken^.Kind := lexerKindRightParen |
|
|
'[': AToken^.Kind := lexerKindLeftSquare |
|
|
']': AToken^.Kind := lexerKindRightSquare |
|
|
'^': AToken^.Kind := lexerKindHat |
|
|
'=': AToken^.Kind := lexerKindEqual |
|
|
'+': AToken^.Kind := lexerKindPlus |
|
|
'/': AToken^.Kind := lexerKindDivision |
|
|
'%': AToken^.Kind := lexerKindRemainder |
|
|
'@': AToken^.Kind := lexerKindAt |
|
|
'|': AToken^.Kind := lexerKindPipe
|
|
END;
|
|
INC(ALexer^.Current)
|
|
END TransitionActionSingle;
|
|
|
|
(* Handle an integer literal. *)
|
|
PROCEDURE TransitionActionInteger(ALexer: PLexer; AToken: PLexerToken);
|
|
BEGIN
|
|
AToken^.Kind := lexerKindInteger
|
|
END TransitionActionInteger;
|
|
|
|
PROCEDURE SetDefaultTransition(CurrentState: TransitionState; DefaultAction: TransitionAction; NextState: TransitionState);
|
|
VAR DefaultTransition: Transition;
|
|
BEGIN
|
|
DefaultTransition.Action := DefaultAction;
|
|
DefaultTransition.NextState := NextState;
|
|
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassInvalid)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassDigit)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassAlpha)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassSpace)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassColon)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassEquals)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassLeftParen)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassRightParen)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassAsterisk)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassUnderscore)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassSingle)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassHex)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassZero)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassX)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassEof)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassDot)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassMinus)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassSingleQuote)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassDoubleQuote)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassGreater)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassLess)] := DefaultTransition;
|
|
Transitions[ORD(CurrentState)][ORD(transitionClassOther)] := DefaultTransition;
|
|
END SetDefaultTransition;
|
|
|
|
(*
|
|
* The transition table describes transitions from one state to another, given
|
|
* a symbol (character class).
|
|
*
|
|
* The table has m rows and n columns, where m is the amount of states and n is
|
|
* the amount of classes. So given the current state and a classified character
|
|
* the table can be used to look up the next state.
|
|
*
|
|
* Each cell is a word long.
|
|
* - The least significant byte of the word is a row number (beginning with 0).
|
|
* It specifies the target state. "ff" means that this is an end state and no
|
|
* transition is possible.
|
|
* - The next byte is the action that should be performed when transitioning.
|
|
* For the meaning of actions see labels in the lex_next function, which
|
|
* handles each action.
|
|
*)
|
|
PROCEDURE InitializeTransitions();
|
|
BEGIN
|
|
(* Start state. *)
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassInvalid)].Action := NIL;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassInvalid)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassDigit)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassDigit)].NextState := transitionStateDecimal;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassAlpha)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassAlpha)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassSpace)].Action := TransitionActionSkip;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassSpace)].NextState := transitionStateStart;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassColon)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassColon)].NextState := transitionStateColon;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassEquals)].Action := TransitionActionSingle;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassEquals)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassLeftParen)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassLeftParen)].NextState := transitionStateLeftParen;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassRightParen)].Action := TransitionActionSingle;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassRightParen)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassAsterisk)].Action := TransitionActionSingle;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassAsterisk)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassUnderscore)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassUnderscore)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassSingle)].Action := TransitionActionSingle;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassSingle)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassHex)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassHex)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassZero)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassZero)].NextState := transitionStateLeadingZero;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassX)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassX)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassEof)].Action := TransitionActionEof;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassEof)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassDot)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassDot)].NextState := transitionStateDot;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassMinus)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassMinus)].NextState := transitionStateMinus;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassSingleQuote)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassSingleQuote)].NextState := transitionStateCharacter;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassDoubleQuote)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassDoubleQuote)].NextState := transitionStateString;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassGreater)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassGreater)].NextState := transitionStateGreater;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassLess)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassLess)].NextState := transitionStateLess;
|
|
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassOther)].Action := NIL;
|
|
Transitions[ORD(transitionStateStart)][ORD(transitionClassOther)].NextState := transitionStateEnd;
|
|
|
|
(* Colon state. *)
|
|
SetDefaultTransition(transitionStateColon, TransitionActionFinalize, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateColon)][ORD(transitionClassEquals)].Action := TransitionActionComposite;
|
|
Transitions[ORD(transitionStateColon)][ORD(transitionClassEquals)].NextState := transitionStateEnd;
|
|
|
|
(* Identifier state. *)
|
|
SetDefaultTransition(transitionStateIdentifier, TransitionActionKeyId, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassDigit)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassDigit)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassAlpha)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassAlpha)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassUnderscore)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassUnderscore)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassHex)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassHex)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassZero)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassZero)].NextState := transitionStateIdentifier;
|
|
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassX)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateIdentifier)][ORD(transitionClassX)].NextState := transitionStateIdentifier;
|
|
|
|
(* Decimal state. *)
|
|
SetDefaultTransition(transitionStateDecimal, TransitionActionInteger, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassDigit)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassDigit)].NextState := transitionStateDecimal;
|
|
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassAlpha)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassAlpha)].NextState := transitionStateDecimalSuffix;
|
|
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassUnderscore)].Action := NIL;
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassUnderscore)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassHex)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassHex)].NextState := transitionStateDecimalSuffix;
|
|
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassZero)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassZero)].NextState := transitionStateDecimal;
|
|
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassX)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateDecimal)][ORD(transitionClassX)].NextState := transitionStateDecimalSuffix;
|
|
|
|
(* Greater state. *)
|
|
SetDefaultTransition(transitionStateGreater, TransitionActionFinalize, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateGreater)][ORD(transitionClassEquals)].Action := TransitionActionComposite;
|
|
Transitions[ORD(transitionStateGreater)][ORD(transitionClassEquals)].NextState := transitionStateEnd;
|
|
|
|
(* Minus state. *)
|
|
SetDefaultTransition(transitionStateMinus, TransitionActionFinalize, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateMinus)][ORD(transitionClassGreater)].Action := TransitionActionComposite;
|
|
Transitions[ORD(transitionStateMinus)][ORD(transitionClassGreater)].NextState := transitionStateEnd;
|
|
|
|
(* Left paren state. *)
|
|
SetDefaultTransition(transitionStateLeftParen, TransitionActionFinalize, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateLeftParen)][ORD(transitionClassAsterisk)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateLeftParen)][ORD(transitionClassAsterisk)].NextState := transitionStateComment;
|
|
|
|
(* Less state. *)
|
|
SetDefaultTransition(transitionStateLess, TransitionActionFinalize, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateLess)][ORD(transitionClassEquals)].Action := TransitionActionComposite;
|
|
Transitions[ORD(transitionStateLess)][ORD(transitionClassEquals)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateLess)][ORD(transitionClassGreater)].Action := TransitionActionComposite;
|
|
Transitions[ORD(transitionStateLess)][ORD(transitionClassGreater)].NextState := transitionStateEnd;
|
|
|
|
(* Hexadecimal after 0x. *)
|
|
SetDefaultTransition(transitionStateDot, TransitionActionFinalize, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateDot)][ORD(transitionClassDot)].Action := TransitionActionComposite;
|
|
Transitions[ORD(transitionStateDot)][ORD(transitionClassDot)].NextState := transitionStateEnd;
|
|
|
|
(* Comment. *)
|
|
SetDefaultTransition(transitionStateComment, TransitionActionAccumulate, transitionStateComment);
|
|
|
|
Transitions[ORD(transitionStateComment)][ORD(transitionClassAsterisk)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateComment)][ORD(transitionClassAsterisk)].NextState := transitionStateClosingComment;
|
|
|
|
Transitions[ORD(transitionStateComment)][ORD(transitionClassEof)].Action := NIL;
|
|
Transitions[ORD(transitionStateComment)][ORD(transitionClassEof)].NextState := transitionStateEnd;
|
|
|
|
(* Closing comment. *)
|
|
SetDefaultTransition(transitionStateClosingComment, TransitionActionAccumulate, transitionStateComment);
|
|
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassInvalid)].Action := NIL;
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassInvalid)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassRightParen)].Action := TransitionActionDelimited;
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassRightParen)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassAsterisk)].Action := TransitionActionAccumulate;
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassAsterisk)].NextState := transitionStateClosingComment;
|
|
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassEof)].Action := NIL;
|
|
Transitions[ORD(transitionStateClosingComment)][ORD(transitionClassEof)].NextState := transitionStateEnd;
|
|
|
|
(* Character. *)
|
|
SetDefaultTransition(transitionStateCharacter, TransitionActionAccumulate, transitionStateCharacter);
|
|
|
|
Transitions[ORD(transitionStateCharacter)][ORD(transitionClassInvalid)].Action := NIL;
|
|
Transitions[ORD(transitionStateCharacter)][ORD(transitionClassInvalid)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateCharacter)][ORD(transitionClassEof)].Action := NIL;
|
|
Transitions[ORD(transitionStateCharacter)][ORD(transitionClassEof)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateCharacter)][ORD(transitionClassSingleQuote)].Action := TransitionActionDelimited;
|
|
Transitions[ORD(transitionStateCharacter)][ORD(transitionClassSingleQuote)].NextState := transitionStateEnd;
|
|
|
|
(* String. *)
|
|
SetDefaultTransition(transitionStateString, TransitionActionAccumulate, transitionStateString);
|
|
|
|
Transitions[ORD(transitionStateString)][ORD(transitionClassInvalid)].Action := NIL;
|
|
Transitions[ORD(transitionStateString)][ORD(transitionClassInvalid)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateString)][ORD(transitionClassEof)].Action := NIL;
|
|
Transitions[ORD(transitionStateString)][ORD(transitionClassEof)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateString)][ORD(transitionClassDoubleQuote)].Action := TransitionActionDelimited;
|
|
Transitions[ORD(transitionStateString)][ORD(transitionClassDoubleQuote)].NextState := transitionStateEnd;
|
|
|
|
(* Leading zero. *)
|
|
SetDefaultTransition(transitionStateLeadingZero, TransitionActionInteger, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassDigit)].Action := NIL;
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassDigit)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassAlpha)].Action := NIL;
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassAlpha)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassUnderscore)].Action := NIL;
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassUnderscore)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassHex)].Action := NIL;
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassHex)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassZero)].Action := NIL;
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassZero)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassX)].Action := NIL;
|
|
Transitions[ORD(transitionStateLeadingZero)][ORD(transitionClassX)].NextState := transitionStateEnd;
|
|
|
|
(* Digit with a character suffix. *)
|
|
SetDefaultTransition(transitionStateDecimalSuffix, TransitionActionInteger, transitionStateEnd);
|
|
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassAlpha)].Action := NIL;
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassAlpha)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassDigit)].Action := NIL;
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassDigit)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassHex)].Action := NIL;
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassHex)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassZero)].Action := NIL;
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassZero)].NextState := transitionStateEnd;
|
|
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassX)].Action := NIL;
|
|
Transitions[ORD(transitionStateDecimalSuffix)][ORD(transitionClassX)].NextState := transitionStateEnd
|
|
END InitializeTransitions;
|
|
|
|
PROCEDURE LexerInitialize(ALexer: PLexer; Input: File);
|
|
BEGIN
|
|
ALexer^.Input := Input;
|
|
ALexer^.Length := 0;
|
|
|
|
ALLOCATE(ALexer^.Buffer, ChunkSize);
|
|
MemZero(ALexer^.Buffer, ChunkSize);
|
|
ALexer^.Size := ChunkSize
|
|
END LexerInitialize;
|
|
|
|
PROCEDURE LexerCurrent(ALexer: PLexer): LexerToken;
|
|
VAR
|
|
CurrentClass: TransitionClass;
|
|
CurrentState: TransitionState;
|
|
CurrentTransition: Transition;
|
|
Result: LexerToken;
|
|
BEGIN
|
|
ALexer^.Current := ALexer^.Start;
|
|
CurrentState := transitionStateStart;
|
|
|
|
WHILE CurrentState <> transitionStateEnd DO
|
|
CurrentClass := Classification[ORD(ALexer^.Current^) + 1];
|
|
|
|
CurrentTransition := Transitions[ORD(CurrentState)][ORD(CurrentClass)];
|
|
IF CurrentTransition.Action <> NIL THEN
|
|
CurrentTransition.Action(ALexer, ADR(Result))
|
|
END;
|
|
CurrentState := CurrentTransition.NextState
|
|
END;
|
|
RETURN Result
|
|
END LexerCurrent;
|
|
|
|
PROCEDURE LexerLex(ALexer: PLexer): LexerToken;
|
|
BEGIN
|
|
IF ALexer^.Length = 0 THEN
|
|
ALexer^.Length := ReadNBytes(ALexer^.Input, ChunkSize, ALexer^.Buffer);
|
|
ALexer^.Current := ALexer^.Buffer
|
|
END;
|
|
ALexer^.Start := ALexer^.Current;
|
|
|
|
RETURN LexerCurrent(ALexer)
|
|
END LexerLex;
|
|
|
|
PROCEDURE LexerDestroy(ALexer: PLexer);
|
|
BEGIN
|
|
DEALLOCATE(ALexer^.Buffer, ALexer^.Size)
|
|
END LexerDestroy;
|
|
|
|
BEGIN
|
|
InitializeClassification();
|
|
InitializeTransitions()
|
|
END Lexer.
|