parser: lazy support

examples/printf.pork

@@ -1,5 +1,5 @@
 import std ffi.printf
 
 export func main() {
-  printf("Hello World: %s\n", "Jolk")
+  printf("Hello World: \n", "Jolk")
 }

frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt

@@ -4,7 +4,6 @@ import gay.pizza.pork.ast.gen.CompilationUnit
 import gay.pizza.pork.ast.gen.ImportDeclaration
 import gay.pizza.pork.parser.DiscardNodeAttribution
 import gay.pizza.pork.parser.Parser
-import gay.pizza.pork.parser.TokenStreamSource
 import gay.pizza.pork.parser.Tokenizer
 
 class World(val importSource: ImportSource) {
@@ -23,8 +22,7 @@ class World(val importSource: ImportSource) {
     }
     val charSource = contentSource.loadAsCharSource(importLocator.path)
     val tokenizer = Tokenizer(charSource)
-    val tokenStream = tokenizer.stream()
-    val parser = Parser(TokenStreamSource(tokenStream), DiscardNodeAttribution)
+    val parser = Parser(tokenizer, DiscardNodeAttribution)
     val unit = parser.parseCompilationUnit()
     internalUnits[stableKey] = unit
     return unit

minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt

@@ -22,14 +22,14 @@ abstract class Tool {
   val rootImportLocator: ImportLocator
     get() = ImportLocator("local", rootFilePath())
 
-  fun tokenize(): LazyTokenSource =
-    LazyTokenSource(Tokenizer(createCharSource()))
+  fun tokenize(): Tokenizer =
+    Tokenizer(createCharSource())
 
   fun parse(attribution: NodeAttribution = DiscardNodeAttribution): CompilationUnit =
-    Parser(TokenStreamSource(tokenize().streamAllRemainingTokens()), attribution).parseCompilationUnit()
+    Parser(tokenize(), attribution).parseCompilationUnit()
 
   fun highlight(scheme: HighlightScheme): List<Highlight> =
-    Highlighter(scheme).highlight(tokenize().streamAllRemainingTokens())
+    Highlighter(scheme).highlight(tokenize().stream())
 
   fun reprint(): String = buildString { visit(Printer(this)) }
 

parser/src/main/kotlin/gay/pizza/pork/parser/DiscardNodeAttribution.kt

@@ -6,6 +6,6 @@ import gay.pizza.pork.ast.gen.NodeType
 object DiscardNodeAttribution : NodeAttribution {
   override fun push(token: Token) {}
   override fun <T : Node> adopt(node: T) {}
-  override fun <T : Node> guarded(type: NodeType?, block: () -> T): T =
+  override fun <T : Node> produce(type: NodeType, block: () -> T): T =
     block()
 }

parser/src/main/kotlin/gay/pizza/pork/parser/LazySkippingTokenSource.kt

@@ -0,0 +1,33 @@
+package gay.pizza.pork.parser
+
+class LazySkippingTokenSource(val source: TokenSource, val skipping: Set<TokenType>) : ParserAwareTokenSource {
+  private var index = 0
+  override val currentIndex: Int
+    get() = index
+
+  private val queue = mutableListOf<Token>()
+
+  override fun next(): Token {
+    needs(1)
+    return queue.removeFirst()
+  }
+
+  override fun peek(): Token {
+    needs(1)
+    return queue.first()
+  }
+
+  override fun peekTypeAhead(ahead: Int): TokenType {
+    needs(ahead + 1)
+    return queue[ahead].type
+  }
+
+  private fun needs(count: Int) {
+    while (queue.size < count) {
+      val token = source.next()
+      if (!skipping.contains(token.type)) {
+        queue.add(token)
+      }
+    }
+  }
+}

parser/src/main/kotlin/gay/pizza/pork/parser/NodeAttribution.kt

@@ -6,5 +6,5 @@ import gay.pizza.pork.ast.gen.NodeType
 interface NodeAttribution {
   fun push(token: Token)
   fun <T: Node> adopt(node: T)
-  fun <T: Node> guarded(type: NodeType?, block: () -> T): T
+  fun <T: Node> produce(type: NodeType, block: () -> T): T
 }

parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt

@@ -4,7 +4,7 @@ import gay.pizza.pork.ast.gen.*
 
 class Parser(source: TokenSource, attribution: NodeAttribution) :
   ParserBase(source, attribution) {
-  override fun parseArgumentSpec(): ArgumentSpec = guarded(NodeType.ArgumentSpec) {
+  override fun parseArgumentSpec(): ArgumentSpec = produce(NodeType.ArgumentSpec) {
     val symbol = parseSymbol()
     ArgumentSpec(symbol, next(TokenType.DotDotDot))
   }
@@ -47,7 +47,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
 
     if (expression is SymbolReference && peek(TokenType.Equals)) {
       val symbolReference = expression
-      expression = guarded(NodeType.SetAssignment) {
+      expression = produce(NodeType.SetAssignment) {
         attribution.adopt(expression)
         expect(TokenType.Equals)
         val value = parseExpression()
@@ -56,7 +56,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     }
 
     if (peek(TokenType.LeftBracket)) {
-      expression = guarded(NodeType.IndexedBy) {
+      expression = produce(NodeType.IndexedBy) {
         attribution.adopt(expression)
         expect(TokenType.LeftBracket)
         val index = parseExpression()
@@ -70,15 +70,15 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
         TokenType.Pipe, TokenType.Caret, TokenType.Equality, TokenType.Inequality, TokenType.Mod,
         TokenType.Rem, TokenType.Lesser, TokenType.Greater, TokenType.LesserEqual, TokenType.GreaterEqual,
         TokenType.And, TokenType.Or)) {
-      guarded(NodeType.InfixOperation) {
+      produce(NodeType.InfixOperation) {
         val infixToken = next()
         val infixOperator = ParserHelpers.convertInfixOperator(infixToken)
         InfixOperation(expression, infixOperator, parseExpression())
       }
     } else expression
   }
 
-  override fun parseBooleanLiteral(): BooleanLiteral = guarded(NodeType.BooleanLiteral) {
+  override fun parseBooleanLiteral(): BooleanLiteral = produce(NodeType.BooleanLiteral) {
     if (next(TokenType.True)) {
       BooleanLiteral(true)
     } else if (next(TokenType.False)) {
@@ -88,12 +88,12 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     }
   }
 
-  override fun parseBreak(): Break = guarded(NodeType.Break) {
+  override fun parseBreak(): Break = produce(NodeType.Break) {
     expect(TokenType.Break)
     Break()
   }
 
-  override fun parseCompilationUnit(): CompilationUnit = guarded(NodeType.CompilationUnit) {
+  override fun parseCompilationUnit(): CompilationUnit = produce(NodeType.CompilationUnit) {
     val declarations = mutableListOf<Declaration>()
     val definitions = mutableListOf<Definition>()
     var declarationAccepted = true
@@ -114,7 +114,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     CompilationUnit(declarations, definitions)
   }
 
-  override fun parseContinue(): Continue = guarded(NodeType.Continue) {
+  override fun parseContinue(): Continue = produce(NodeType.Continue) {
     expect(TokenType.Continue)
     Continue()
   }
@@ -187,11 +187,11 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     ForIn(forInItem, value, block)
   }
 
-  override fun parseForInItem(): ForInItem = guarded(NodeType.ForInItem) {
+  override fun parseForInItem(): ForInItem = produce(NodeType.ForInItem) {
     ForInItem(parseSymbol())
   }
 
-  override fun parseFunctionCall(): FunctionCall = guarded(NodeType.FunctionCall) {
+  override fun parseFunctionCall(): FunctionCall = produce(NodeType.FunctionCall) {
     val symbol = parseSymbol()
     expect(TokenType.LeftParentheses)
     val arguments = collect(TokenType.RightParentheses, TokenType.Comma) {
@@ -201,7 +201,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     FunctionCall(symbol, arguments)
   }
 
-  override fun parseFunctionDefinition(): FunctionDefinition = guarded(NodeType.FunctionDefinition) {
+  override fun parseFunctionDefinition(): FunctionDefinition = produce(NodeType.FunctionDefinition) {
     val modifiers = parseDefinitionModifiers()
     expect(TokenType.Func)
     val name = parseSymbol()
@@ -236,22 +236,22 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     ImportDeclaration(form, parseImportPath())
   }
 
-  override fun parseImportPath(): ImportPath = guarded(NodeType.ImportPath) {
+  override fun parseImportPath(): ImportPath = produce(NodeType.ImportPath) {
     val components = oneAndContinuedBy(TokenType.Dot) {
       parseSymbol()
     }
     ImportPath(components)
   }
 
-  override fun parseIndexedBy(): IndexedBy = guarded(NodeType.IndexedBy) {
+  override fun parseIndexedBy(): IndexedBy = produce(NodeType.IndexedBy) {
     val expression = parseExpression()
     expect(TokenType.LeftBracket)
     val index = parseExpression()
     expect(TokenType.RightBracket)
     IndexedBy(expression, index)
   }
 
-  override fun parseInfixOperation(): InfixOperation = guarded(NodeType.InfixOperation) {
+  override fun parseInfixOperation(): InfixOperation = produce(NodeType.InfixOperation) {
     val infixToken = next()
     val infixOperator = ParserHelpers.convertInfixOperator(infixToken)
     InfixOperation(parseExpression(), infixOperator, parseExpression())
@@ -282,7 +282,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     LetAssignment(symbol, value)
   }
 
-  override fun parseLetDefinition(): LetDefinition = guarded(NodeType.LetDefinition) {
+  override fun parseLetDefinition(): LetDefinition = produce(NodeType.LetDefinition) {
     val definitionModifiers = parseDefinitionModifiers()
     expect(TokenType.Let)
     val name = parseSymbol()
@@ -330,22 +330,22 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     PrefixOperation(ParserHelpers.convertPrefixOperator(it), parseExpression())
   }
 
-  override fun parseSetAssignment(): SetAssignment = guarded(NodeType.SetAssignment) {
+  override fun parseSetAssignment(): SetAssignment = produce(NodeType.SetAssignment) {
     val symbol = parseSymbol()
     expect(TokenType.Equals)
     val value = parseExpression()
     SetAssignment(symbol, value)
   }
 
-  override fun parseStringLiteral(): StringLiteral = guarded(NodeType.StringLiteral) {
+  override fun parseStringLiteral(): StringLiteral = produce(NodeType.StringLiteral) {
     expect(TokenType.Quote)
     val stringLiteralToken = expect(TokenType.StringLiteral)
     expect(TokenType.Quote)
     val content = StringEscape.unescape(stringLiteralToken.text)
     StringLiteral(content)
   }
 
-  override fun parseSuffixOperation(): SuffixOperation = guarded(NodeType.SuffixOperation) {
+  override fun parseSuffixOperation(): SuffixOperation = produce(NodeType.SuffixOperation) {
     val reference = parseSymbolReference()
     expect(TokenType.PlusPlus, TokenType.MinusMinus) {
       SuffixOperation(ParserHelpers.convertSuffixOperator(it), reference)
@@ -364,7 +364,7 @@ class Parser(source: TokenSource, attribution: NodeAttribution) :
     Symbol(it.text)
   }
 
-  override fun parseSymbolReference(): SymbolReference = guarded(NodeType.SymbolReference) {
+  override fun parseSymbolReference(): SymbolReference = produce(NodeType.SymbolReference) {
     SymbolReference(parseSymbol())
   }
 

parser/src/main/kotlin/gay/pizza/pork/parser/ParserAwareTokenSource.kt

@@ -0,0 +1,3 @@
+package gay.pizza.pork.parser
+
+interface ParserAwareTokenSource : TokenSource

parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt

@@ -5,15 +5,19 @@ import gay.pizza.pork.ast.gen.NodeParser
 import gay.pizza.pork.ast.gen.NodeType
 
 abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) : NodeParser {
-  val source: TokenSource = source.ignoringParserIgnoredTypes()
+  val source: TokenSource = if (source is ParserAwareTokenSource) {
+    source
+  } else {
+    LazySkippingTokenSource(source, TokenType.ParserIgnoredTypes)
+  }
 
   @Suppress("NOTHING_TO_INLINE")
-  protected inline fun <T: Node> guarded(type: NodeType? = null, noinline block: () -> T): T =
-    attribution.guarded(type, block)
+  protected inline fun <T: Node> produce(type: NodeType, noinline block: () -> T): T =
+    attribution.produce(type, block)
 
   @Suppress("NOTHING_TO_INLINE")
-  protected inline fun <T: Node> expect(type: NodeType? = null, vararg tokenTypes: TokenType, noinline block: (Token) -> T): T =
-    guarded(type) {
+  protected inline fun <T: Node> expect(type: NodeType, vararg tokenTypes: TokenType, noinline block: (Token) -> T): T =
+    produce(type) {
       block(expect(*tokenTypes))
     }
 

parser/src/main/kotlin/gay/pizza/pork/parser/ParserNodeAttribution.kt

@@ -23,7 +23,7 @@ open class ParserNodeAttribution : NodeAttribution {
     }
   }
 
-  override fun <T : Node> guarded(type: NodeType?, block: () -> T): T {
+  override fun <T : Node> produce(type: NodeType, block: () -> T): T {
     var store = mutableListOf<Token>()
     current = store
     stack.add(store)

parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt

@@ -2,13 +2,13 @@ package gay.pizza.pork.parser
 
 class SourceIndexCharSource(val delegate: CharSource) : CharSource by delegate {
   private var currentLineIndex = 1
-  private var currentLineColumn = 0
+  private var currentLineColumn = 1
 
   override fun next(): Char {
     val char = delegate.next()
     if (char == '\n') {
       currentLineIndex++
-      currentLineColumn = 0
+      currentLineColumn = 1
     }
     currentLineColumn++
     return char

parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt

@@ -1,7 +1,7 @@
 package gay.pizza.pork.parser
 
 object StringCharConsumer : CharConsumer {
-  override fun consume(type: TokenType, tokenizer: Tokenizer): String? {
+  override fun consume(type: TokenType, tokenizer: Tokenizer): String {
     val buffer = StringBuilder()
     var escape = false
     while (true) {
@@ -15,6 +15,10 @@ object StringCharConsumer : CharConsumer {
         break
       }
 
+      if (escape) {
+        escape = false
+      }
+
       buffer.append(tokenizer.source.next())
 
       if (char == '\\') {

parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt

@@ -2,6 +2,7 @@ package gay.pizza.pork.parser
 
 interface TokenSource : PeekableSource<Token> {
   fun peekTypeAhead(ahead: Int): TokenType
+
   fun consumeAllRemainingTokens(): List<Token> {
     val tokens = mutableListOf<Token>()
     while (true) {
@@ -14,9 +15,5 @@ interface TokenSource : PeekableSource<Token> {
     return tokens
   }
 
-  fun streamAllRemainingTokens(): TokenStream =
-    TokenStream(consumeAllRemainingTokens().filter { !TokenType.ParserIgnoredTypes.contains(it.type) })
-
-  fun ignoringParserIgnoredTypes(): TokenSource =
-    TokenStreamSource(streamAllRemainingTokens())
+  fun stream(): TokenStream = TokenStream(consumeAllRemainingTokens())
 }

parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt

@@ -107,7 +107,7 @@ enum class TokenType(vararg val properties: TokenTypeProperty) {
     val CharMatches = entries.filter { item -> item.charMatch != null }
     val CharConsumes = entries.filter { item -> item.charConsume != null }
 
-    val ParserIgnoredTypes: Array<TokenType> = arrayOf(
+    val ParserIgnoredTypes: Set<TokenType> = setOf(
       Whitespace,
       BlockComment,
       LineComment