wip: Fix lexer

broxus · Aug 27, 2023 · acde05d · acde05d
1 parent 4d8e76d
commit acde05d
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 7 deletions.
diff --git a/src/core/cont.rs b/src/core/cont.rs
@@ -95,7 +95,7 @@ impl ContImpl for InterpreterCont {
 
         'source_block: loop {
             'token: {
-                let mut rewind = 0;
+                let mut rewind = None;
                 let entry = 'entry: {
                     let Some(token) = ctx.input.scan_word()? else {
                         if ctx.input.pop_source_block() {
@@ -120,7 +120,7 @@ impl ContImpl for InterpreterCont {
                         while !word.is_empty() {
                             word.pop();
                             if let Some(entry) = ctx.dicts.lookup(&word, false)? {
-                                rewind = token.len() - word.len();
+                                rewind = Some(token.len() - word.len());
                                 return Ok(Some(entry));
                             }
                         }
@@ -144,7 +144,12 @@ impl ContImpl for InterpreterCont {
 
                     anyhow::bail!("Undefined word `{token}`");
                 };
-                ctx.input.rewind(rewind);
+
+                if let Some(rewind) = rewind {
+                    ctx.input.rewind(rewind);
+                } else {
+                    ctx.input.scan_skip_whitespace()?;
+                }
 
                 if entry.active {
                     ctx.next = SeqCont::make(

diff --git a/src/core/lexer.rs b/src/core/lexer.rs
@@ -133,19 +133,28 @@ pub struct LexerPosition<'a> {
 
 pub trait Delimiter {
     fn delim(&mut self, c: char) -> bool;
+    fn is_eof(&self) -> bool;
 }
 
 impl<T: FnMut(char) -> bool> Delimiter for T {
     fn delim(&mut self, c: char) -> bool {
         (self)(c)
     }
+
+    fn is_eof(&self) -> bool {
+        false
+    }
 }
 
 impl Delimiter for char {
     #[inline]
     fn delim(&mut self, c: char) -> bool {
         *self == c
     }
+
+    fn is_eof(&self) -> bool {
+        *self as u32 == 0
+    }
 }
 
 struct SourceBlockState {
@@ -206,8 +215,13 @@ impl SourceBlockState {
 
         let end = self.line_offset;
 
+        let is_last = end == self.line.len() && p.is_eof();
+        found |= is_last;
+
         Ok(if found && end >= start {
-            self.skip_symbol();
+            if !is_last {
+                self.skip_symbol();
+            }
             Some(&self.line[start..end])
         } else {
             None

diff --git a/src/modules/control.rs b/src/modules/control.rs
@@ -227,7 +227,7 @@ impl Control {
 
     #[cmd(name = "(word-prefix-find)")]
     fn interpret_word_prefix_find(ctx: &mut Context) -> Result<()> {
-        let mut rewind = 0;
+        let mut rewind = None;
         let (word, entry) = 'entry: {
             let Some(token) = ctx.input.scan_word()? else {
                 ctx.stack.push(String::new())?;
@@ -248,7 +248,7 @@ impl Control {
             while !word.is_empty() {
                 word.pop();
                 if let Some(entry) = ctx.dicts.lookup(&word, false)? {
-                    rewind = token.len() - word.len();
+                    rewind = Some(token.len() - word.len());
                     break 'entry (word, Some(entry));
                 }
             }
@@ -259,7 +259,14 @@ impl Control {
             //ctx.input.scan_skip_whitespace()?;
             (word, None)
         };
-        ctx.input.rewind(rewind);
+
+        println!("(WORD): |{word}|");
+
+        if let Some(rewind) = rewind {
+            ctx.input.rewind(rewind);
+        } else {
+            ctx.input.scan_skip_whitespace()?;
+        }
 
         match entry {
             None => {
@@ -279,6 +286,8 @@ impl Control {
         let cont = ctx.stack.pop_cont()?;
         let word = ctx.input.scan_word()?.ok_or(UnexpectedEof)?.to_owned();
 
+        println!("CREATE: |{word}|");
+
         define_word(
             &mut ctx.dicts.current,
             word,
@@ -304,6 +313,8 @@ impl Control {
         };
         let word = ctx.stack.pop_string_owned()?;
         let cont = ctx.stack.pop_cont_owned()?;
+        println!("(CREATE): |{word}|");
+
         define_word(&mut ctx.dicts.current, word, cont, mode)
     }