Handle ASCII codes > 128 in the tokenizer

source/Lexer.mod

@@ -72,6 +72,8 @@ VAR
   transitions: ARRAY[1..16] OF TransitionClasses;
 
 PROCEDURE initialize_classification();
+VAR
+  i: CARDINAL;
 BEGIN
   classification[1] := transitionClassEof; (* NUL *)
   classification[2] := transitionClassInvalid; (* SOH *)
@@ -200,23 +202,29 @@ BEGIN
   classification[125] := transitionClassSingle; (* | *)
   classification[126] := transitionClassOther; (* } *)
   classification[127] := transitionClassSingle; (* ~ *)
-  classification[128] := transitionClassInvalid (* DEL *)
+  classification[128] := transitionClassInvalid; (* DEL *)
+
+  i := 129;
+  WHILE i <= 256 DO
+    classification[i] := transitionClassOther;
+    i := i + 1
+  END
 END initialize_classification;
 PROCEDURE compare_keyword(Keyword: ARRAY OF CHAR;  TokenStart: PLexerBuffer;  TokenEnd: PLexerBuffer): BOOLEAN;
 VAR
-  Result: BOOLEAN;
+  result: BOOLEAN;
   Index: CARDINAL;
 BEGIN
   Index := 0;
-  Result := TRUE;
+  result := TRUE;
 
-  WHILE (Index < Length(Keyword)) AND (TokenStart <> TokenEnd) AND Result DO
-    Result := (Keyword[Index] = TokenStart^) OR (Lower(Keyword[Index]) = TokenStart^);
+  WHILE (Index < Length(Keyword)) AND (TokenStart <> TokenEnd) AND result DO
+    result := (Keyword[Index] = TokenStart^) OR (Lower(Keyword[Index]) = TokenStart^);
     INC(TokenStart);
     INC(Index)
   END;
-  Result := (Index = Length(Keyword)) AND (TokenStart = TokenEnd) AND Result;
-  RETURN Result
+  result := (Index = Length(Keyword)) AND (TokenStart = TokenEnd) AND result;
+  RETURN result
 END compare_keyword;
 (* Reached the end of file. *)
 PROCEDURE transition_action_eof(lexer: PLexer;  AToken: PLexerToken);
@@ -747,7 +755,7 @@ VAR
   CurrentClass: TransitionClass;
   CurrentState: TransitionState;
   CurrentTransition: Transition;
-  Result: LexerToken;
+  result: LexerToken;
 BEGIN
   lexer^.Current := lexer^.Start;
   CurrentState := transitionStateStart;
@@ -757,15 +765,15 @@ BEGIN
 
     CurrentTransition := transitions[ORD(CurrentState) + 1][ORD(CurrentClass) + 1];
     IF CurrentTransition.Action <> NIL THEN
-      CurrentTransition.Action(lexer, ADR(Result))
+      CurrentTransition.Action(lexer, ADR(result))
     END;
     CurrentState := CurrentTransition.NextState
   END;
-  RETURN Result
+  RETURN result
 END lexer_current;
 PROCEDURE lexer_lex(lexer: PLexer): LexerToken;
 VAR
-  Result: LexerToken;
+  result: LexerToken;
 BEGIN
   IF lexer^.Length = 0 THEN
     lexer^.Length := ReadNBytes(lexer^.Input, CHUNK_SIZE, lexer^.Buffer);
@@ -773,8 +781,8 @@ BEGIN
   END;
   lexer^.Start := lexer^.Current;
 
-  Result := lexer_current(lexer);
-  RETURN Result
+  result := lexer_current(lexer);
+  RETURN result
 END lexer_lex;
 PROCEDURE lexer_destroy(lexer: PLexer);
 BEGIN