From 40701008f04f2242ab69bfb4bc4f376e6b75429a Mon Sep 17 00:00:00 2001
From: Eugen Wissner <belka@caraus.de>
Date: Tue, 6 May 2025 23:58:46 +0200
Subject: Replace _read_token with the lexer

---
 boot/tokenizer.s | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

(limited to 'boot/tokenizer.s')

diff --git a/boot/tokenizer.s b/boot/tokenizer.s
index bf8e443..647a3b6 100644
--- a/boot/tokenizer.s
+++ b/boot/tokenizer.s
@@ -268,7 +268,7 @@ transitions:
 
 	.word 0x08ff, 0x0103, 0x00ff, 0x08ff, 0x08ff, 0x08ff, 0x08ff, 0x08ff
 	.word 0x08ff, 0x00ff, 0x08ff, 0x00ff, 0x0103, 0x00ff, 0x08ff, 0x08ff
-	.word 0x08ff, 0x08ff, 0x08ff, 0x08ff # 0x03 Integer
+	.word 0x08ff, 0x08ff, 0x08ff, 0x08ff # 0x03 Decimal
 
 	.word 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x04ff, 0x02ff, 0x02ff
 	.word 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff
@@ -286,9 +286,9 @@ transitions:
 	.word 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff
 	.word 0x02ff, 0x02ff, 0x02ff, 0x04ff # 0x07 Less
 
-	.word 0x02ff, 0x0102, 0x0102, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff
-	.word 0x02ff, 0x0102, 0x02ff, 0x0102, 0x0102, 0x0102, 0x02ff, 0x02ff
-	.word 0x02ff, 0x02ff, 0x02ff, 0x02ff # 0x08 Dot
+	.word 0x08ff, 0x0108, 0x00ff, 0x08ff, 0x08ff, 0x08ff, 0x08ff, 0x08ff
+	.word 0x08ff, 0x00ff, 0x08ff, 0x0108, 0x0108, 0x00ff, 0x08ff, 0x08ff
+	.word 0x08ff, 0x08ff, 0x08ff, 0x08ff # 0x08 Hexadecimal after 0x.
 
 	.word 0x0109, 0x0109, 0x0109, 0x0109, 0x0109, 0x0109, 0x0109, 0x0109
 	.word 0x010a, 0x0109, 0x0109, 0x0109, 0x0109, 0x0109, 0x00ff, 0x0109
@@ -302,13 +302,13 @@ transitions:
 	.word 0x010b, 0x010b, 0x010b, 0x010b, 0x010b, 0x010b, 0x010b, 0x0110
 	.word 0x010b, 0x04ff, 0x010b, 0x010b # 0x0b String
 
-	.word 0x02ff, 0x00ff, 0x00ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff
-	.word 0x02ff, 0x02ff, 0x02ff, 0x00ff, 0x00ff, 0x010d, 0x02ff, 0x02ff
-	.word 0x02ff, 0x02ff, 0x02ff, 0x02ff # 0x0c Zero
+	.word 0x08ff, 0x00ff, 0x00ff, 0x08ff, 0x08ff, 0x08ff, 0x08ff, 0x08ff
+	.word 0x08ff, 0x00ff, 0x08ff, 0x00ff, 0x00ff, 0x010d, 0x08ff, 0x08ff
+	.word 0x08ff, 0x08ff, 0x08ff, 0x08ff # 0x0c Leading zero
 
-	.word 0x02ff, 0x010d, 0x00ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff, 0x02ff
-	.word 0x02ff, 0x02ff, 0x02ff, 0x010d, 0x010d, 0x00ff, 0x2ff, 0x02ff
-	.word 0x00ff, 0x02ff, 0x02ff, 0x02ff # 0x0d Hexadecimal
+	.word 0x00ff, 0x0108, 0x00ff, 0x00ff, 0x00ff, 0x00ff, 0x00ff, 0x00ff
+	.word 0x00ff, 0x00ff, 0x00ff, 0x0108, 0x0108, 0x00ff, 0x00ff, 0x00ff
+	.word 0x00ff, 0x00ff, 0x00ff, 0x00ff # 0x0d Starting hexadecimal
 
 .section .text
 
@@ -592,14 +592,14 @@ _tokenize_next:
 	j .Ltokenize_next_end
 
 .Ltokenize_next_integer:
-	lw a1, 12(sp)
-	sub a0, s1, a1
-	sw a0, 8(sp)
-	sw a0, 4(sp)
-	lw a0, 0(sp)
-	addi a1, sp, 4
-	li a2, 12
-	call _memcpy
+	lw t0, 0(sp)
+	li t1, TOKEN_INTEGER
+	sw t1, 0(t0)
+	lw t1, 12(sp)
+	sw t1, 8(t0)
+	sub t1, s1, t1
+	sw t1, 4(t0)
+
 	j .Ltokenize_next_end
 
 .Ltokenize_next_end:
-- 
cgit v1.2.3