build: move parser and tokenizer into separate modules

ffi/src/main/kotlin/gay/pizza/pork/ffi/JavaAutogenContentSource.kt

@@ -2,9 +2,9 @@ package gay.pizza.pork.ffi
 
 import gay.pizza.pork.ast.gen.visit
 import gay.pizza.pork.frontend.ContentSource
-import gay.pizza.pork.parser.CharSource
+import gay.pizza.pork.tokenizer.CharSource
 import gay.pizza.pork.parser.Printer
-import gay.pizza.pork.parser.StringCharSource
+import gay.pizza.pork.tokenizer.StringCharSource
 
 object JavaAutogenContentSource : ContentSource {
   override fun loadAsCharSource(path: String): CharSource {

frontend/src/main/kotlin/gay/pizza/pork/frontend/ContentSource.kt

@@ -1,6 +1,6 @@
 package gay.pizza.pork.frontend
 
-import gay.pizza.pork.parser.CharSource
+import gay.pizza.pork.tokenizer.CharSource
 
 interface ContentSource {
   fun loadAsCharSource(path: String): CharSource

frontend/src/main/kotlin/gay/pizza/pork/frontend/FsContentSource.kt

@@ -3,8 +3,8 @@ package gay.pizza.pork.frontend
 import gay.pizza.dough.fs.FsPath
 import gay.pizza.dough.fs.PlatformFsProvider
 import gay.pizza.dough.fs.readString
-import gay.pizza.pork.parser.CharSource
-import gay.pizza.pork.parser.StringCharSource
+import gay.pizza.pork.tokenizer.CharSource
+import gay.pizza.pork.tokenizer.StringCharSource
 
 class FsContentSource(val root: FsPath) : ContentSource {
   override fun loadAsCharSource(path: String): CharSource =

frontend/src/main/kotlin/gay/pizza/pork/frontend/World.kt

@@ -4,7 +4,7 @@ import gay.pizza.pork.ast.gen.CompilationUnit
 import gay.pizza.pork.ast.gen.ImportDeclaration
 import gay.pizza.pork.parser.DiscardNodeAttribution
 import gay.pizza.pork.parser.Parser
-import gay.pizza.pork.parser.Tokenizer
+import gay.pizza.pork.tokenizer.Tokenizer
 
 class World(val importSource: ImportSource) {
   private val internalUnits = mutableMapOf<String, CompilationUnit>()

minimal/src/main/kotlin/gay/pizza/pork/minimal/FileTool.kt

@@ -4,8 +4,8 @@ import gay.pizza.dough.fs.FsPath
 import gay.pizza.dough.fs.readString
 import gay.pizza.pork.frontend.ContentSource
 import gay.pizza.pork.frontend.FsContentSource
-import gay.pizza.pork.parser.CharSource
-import gay.pizza.pork.parser.StringCharSource
+import gay.pizza.pork.tokenizer.CharSource
+import gay.pizza.pork.tokenizer.StringCharSource
 
 class FileTool(val path: FsPath) : Tool() {
   override fun createCharSource(): CharSource =

minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt

@@ -13,6 +13,7 @@ import gay.pizza.pork.frontend.DynamicImportSource
 import gay.pizza.pork.frontend.World
 import gay.pizza.pork.parser.*
 import gay.pizza.pork.stdlib.PorkStdlib
+import gay.pizza.pork.tokenizer.*
 
 abstract class Tool {
   abstract fun createCharSource(): CharSource

parser/build.gradle.kts

@@ -4,6 +4,7 @@ plugins {
 
 dependencies {
   api(project(":ast"))
+  api(project(":tokenizer"))
 
   implementation(project(":common"))
 }

parser/src/main/kotlin/gay/pizza/pork/parser/DiscardNodeAttribution.kt

@@ -2,6 +2,7 @@ package gay.pizza.pork.parser
 
 import gay.pizza.pork.ast.gen.Node
 import gay.pizza.pork.ast.gen.NodeType
+import gay.pizza.pork.tokenizer.Token
 
 object DiscardNodeAttribution : NodeAttribution {
   override fun push(token: Token) {}

parser/src/main/kotlin/gay/pizza/pork/parser/LazySkippingTokenSource.kt

@@ -1,5 +1,9 @@
 package gay.pizza.pork.parser
 
+import gay.pizza.pork.tokenizer.Token
+import gay.pizza.pork.tokenizer.TokenSource
+import gay.pizza.pork.tokenizer.TokenType
+
 class LazySkippingTokenSource(val source: TokenSource, val skipping: Set<TokenType>) : ParserAwareTokenSource {
   private var index = 0
   override val currentIndex: Int

parser/src/main/kotlin/gay/pizza/pork/parser/NodeAttribution.kt

@@ -2,6 +2,7 @@ package gay.pizza.pork.parser
 
 import gay.pizza.pork.ast.gen.Node
 import gay.pizza.pork.ast.gen.NodeType
+import gay.pizza.pork.tokenizer.Token
 
 interface NodeAttribution {
   fun push(token: Token)

parser/src/main/kotlin/gay/pizza/pork/parser/Parser.kt

@@ -1,6 +1,8 @@
 package gay.pizza.pork.parser
 
 import gay.pizza.pork.ast.gen.*
+import gay.pizza.pork.tokenizer.TokenSource
+import gay.pizza.pork.tokenizer.TokenType
 
 class Parser(source: TokenSource, attribution: NodeAttribution) :
   ParserBase(source, attribution) {

parser/src/main/kotlin/gay/pizza/pork/parser/ParserAttributes.kt

@@ -4,6 +4,7 @@ import gay.pizza.pork.ast.gen.Node
 import gay.pizza.pork.ast.gen.NodeCoalescer
 import gay.pizza.pork.ast.gen.data
 import gay.pizza.pork.ast.gen.visit
+import gay.pizza.pork.tokenizer.Token
 
 data class ParserAttributes(val tokens: List<Token>) {
   companion object {

parser/src/main/kotlin/gay/pizza/pork/parser/ParserAwareTokenSource.kt

@@ -1,3 +1,5 @@
 package gay.pizza.pork.parser
 
+import gay.pizza.pork.tokenizer.TokenSource
+
 interface ParserAwareTokenSource : TokenSource

parser/src/main/kotlin/gay/pizza/pork/parser/ParserBase.kt

@@ -3,6 +3,7 @@ package gay.pizza.pork.parser
 import gay.pizza.pork.ast.gen.Node
 import gay.pizza.pork.ast.gen.NodeParser
 import gay.pizza.pork.ast.gen.NodeType
+import gay.pizza.pork.tokenizer.*
 
 abstract class ParserBase(source: TokenSource, val attribution: NodeAttribution) : NodeParser {
   val source: TokenSource = if (source is ParserAwareTokenSource) {

parser/src/main/kotlin/gay/pizza/pork/parser/ParserHelpers.kt

@@ -3,6 +3,8 @@ package gay.pizza.pork.parser
 import gay.pizza.pork.ast.gen.InfixOperator
 import gay.pizza.pork.ast.gen.PrefixOperator
 import gay.pizza.pork.ast.gen.SuffixOperator
+import gay.pizza.pork.tokenizer.Token
+import gay.pizza.pork.tokenizer.TokenType
 
 internal object ParserHelpers {
   fun convertInfixOperator(token: Token): InfixOperator = when (token.type) {

parser/src/main/kotlin/gay/pizza/pork/parser/ParserNodeAttribution.kt

@@ -3,6 +3,7 @@ package gay.pizza.pork.parser
 import gay.pizza.pork.ast.gen.Node
 import gay.pizza.pork.ast.gen.NodeType
 import gay.pizza.pork.ast.gen.data
+import gay.pizza.pork.tokenizer.Token
 
 open class ParserNodeAttribution : NodeAttribution {
   private val stack = mutableListOf<MutableList<Token>>()

settings.gradle.kts

@@ -4,6 +4,7 @@ includeBuild("buildext")
 
 include(
   ":common",
+  ":tokenizer",
   ":ast",
   ":parser",
   ":frontend",

stdlib/src/main/kotlin/gay/pizza/pork/stdlib/PorkStdlib.kt

@@ -1,8 +1,8 @@
 package gay.pizza.pork.stdlib
 
 import gay.pizza.pork.frontend.ContentSource
-import gay.pizza.pork.parser.CharSource
-import gay.pizza.pork.parser.StringCharSource
+import gay.pizza.pork.tokenizer.CharSource
+import gay.pizza.pork.tokenizer.StringCharSource
 
 object PorkStdlib : ContentSource {
   private val stdlibClass = PorkStdlib::class.java

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkBraceMatcher.kt

@@ -4,7 +4,7 @@ import com.intellij.lang.BracePair
 import com.intellij.lang.PairedBraceMatcher
 import com.intellij.psi.PsiFile
 import com.intellij.psi.tree.IElementType
-import gay.pizza.pork.parser.TokenType
+import gay.pizza.pork.tokenizer.TokenType
 
 class PorkBraceMatcher : PairedBraceMatcher {
   override fun getPairs(): Array<BracePair> = arrayOf(

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkElementTypes.kt

@@ -3,7 +3,7 @@ package gay.pizza.pork.idea
 import com.intellij.psi.tree.IElementType
 import com.intellij.psi.tree.TokenSet
 import gay.pizza.pork.ast.gen.NodeType
-import gay.pizza.pork.parser.TokenType
+import gay.pizza.pork.tokenizer.TokenType
 
 object PorkElementTypes {
   private val tokenTypeToElementType = mutableMapOf<TokenType, IElementType>()

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkLexer.kt

@@ -3,10 +3,10 @@ package gay.pizza.pork.idea
 import com.intellij.lexer.LexerBase
 import com.intellij.openapi.progress.ProcessCanceledException
 import com.intellij.psi.tree.IElementType
-import gay.pizza.pork.parser.BadCharacterError
-import gay.pizza.pork.parser.StringCharSource
-import gay.pizza.pork.parser.Tokenizer
-import gay.pizza.pork.parser.UnterminatedTokenError
+import gay.pizza.pork.tokenizer.BadCharacterError
+import gay.pizza.pork.tokenizer.StringCharSource
+import gay.pizza.pork.tokenizer.Tokenizer
+import gay.pizza.pork.tokenizer.UnterminatedTokenError
 import com.intellij.psi.TokenType as PsiTokenType
 
 class PorkLexer : LexerBase() {

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParameterInfoHandler.kt

@@ -11,7 +11,7 @@ import com.intellij.psi.util.elementsAtOffsetUp
 import gay.pizza.pork.idea.psi.gen.ArgumentSpecElement
 import gay.pizza.pork.idea.psi.gen.FunctionCallElement
 import gay.pizza.pork.idea.psi.gen.FunctionDefinitionElement
-import gay.pizza.pork.parser.TokenType
+import gay.pizza.pork.tokenizer.TokenType
 
 @Suppress("UnstableApiUsage")
 class PorkParameterInfoHandler : ParameterInfoHandler<FunctionCallElement, FunctionDefinitionElement> {

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkParserDefinition.kt

@@ -11,7 +11,7 @@ import com.intellij.psi.PsiFile
 import com.intellij.psi.tree.IFileElementType
 import com.intellij.psi.tree.TokenSet
 import gay.pizza.pork.idea.psi.gen.PorkElementFactory
-import gay.pizza.pork.parser.TokenType
+import gay.pizza.pork.tokenizer.TokenType
 
 class PorkParserDefinition : ParserDefinition {
   val fileElementType = IFileElementType(PorkLanguage)

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PorkSyntaxHighlighter.kt

@@ -6,8 +6,8 @@ import com.intellij.openapi.editor.colors.TextAttributesKey
 import com.intellij.openapi.fileTypes.SyntaxHighlighter
 import com.intellij.openapi.fileTypes.SyntaxHighlighterBase
 import com.intellij.psi.tree.IElementType
-import gay.pizza.pork.parser.TokenFamily
-import gay.pizza.pork.parser.TokenType
+import gay.pizza.pork.tokenizer.TokenFamily
+import gay.pizza.pork.tokenizer.TokenType
 
 object PorkSyntaxHighlighter : SyntaxHighlighter {
   override fun getHighlightingLexer(): Lexer {

support/pork-idea/src/main/kotlin/gay/pizza/pork/idea/PsiBuilderTokenSource.kt

@@ -1,7 +1,10 @@
 package gay.pizza.pork.idea
 
 import com.intellij.lang.PsiBuilder
-import gay.pizza.pork.parser.*
+import gay.pizza.pork.parser.ParserAwareTokenSource
+import gay.pizza.pork.tokenizer.SourceIndex
+import gay.pizza.pork.tokenizer.Token
+import gay.pizza.pork.tokenizer.TokenType
 import com.intellij.psi.TokenType as PsiTokenType
 
 @Suppress("UnstableApiUsage")

tokenizer/build.gradle.kts

@@ -0,0 +1,7 @@
+plugins {
+  id("gay.pizza.pork.module")
+}
+
+dependencies {
+  implementation(project(":common"))
+}

parser/src/main/kotlin/gay/pizza/pork/parser/AnsiHighlightScheme.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/AnsiHighlightScheme.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 open class AnsiHighlightScheme : HighlightScheme {
   override fun highlight(token: Token): Highlight {

tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/BadCharacterError.kt

@@ -0,0 +1,5 @@
+package gay.pizza.pork.tokenizer
+
+class BadCharacterError(char: Char, sourceIndex: SourceIndex, state: TokenizerState) : TokenizeError(
+  "Failed to produce token for '${char}' at $sourceIndex in state $state"
+)

parser/src/main/kotlin/gay/pizza/pork/parser/CharConsumer.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharConsumer.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 interface CharConsumer {
   fun consume(type: TokenType, tokenizer: Tokenizer): String?

parser/src/main/kotlin/gay/pizza/pork/parser/CharMatcher.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharMatcher.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 fun interface CharMatcher {
   fun valid(char: Char, index: Int): Boolean

parser/src/main/kotlin/gay/pizza/pork/parser/CharSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSource.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 interface CharSource : PeekableSource<Char> {
   fun peek(index: Int): Char

parser/src/main/kotlin/gay/pizza/pork/parser/CharSourceExtensions.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/CharSourceExtensions.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 fun CharSource.readToString(): String = buildString {
   while (peek() != CharSource.EndOfFile) {

parser/src/main/kotlin/gay/pizza/pork/parser/ExpectedTokenError.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ExpectedTokenError.kt

@@ -1,6 +1,6 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
-class ExpectedTokenError(got: Token, sourceIndex: SourceIndex, vararg expectedTypes: TokenType) : ParseError(
+class ExpectedTokenError(got: Token, sourceIndex: SourceIndex, vararg expectedTypes: TokenType) : TokenizeError(
   message(got, sourceIndex, expectedTypes)
 ) {
   companion object {

parser/src/main/kotlin/gay/pizza/pork/parser/Highlight.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlight.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 class Highlight(val token: Token, val text: String? = null) {
   override fun toString(): String = text ?: token.text

parser/src/main/kotlin/gay/pizza/pork/parser/HighlightScheme.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/HighlightScheme.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 interface HighlightScheme {
   fun highlight(token: Token): Highlight

parser/src/main/kotlin/gay/pizza/pork/parser/Highlighter.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 class Highlighter(val scheme: HighlightScheme) {
   fun highlight(stream: TokenStream): List<Highlight> =

parser/src/main/kotlin/gay/pizza/pork/parser/MatchedCharConsumer.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/MatchedCharConsumer.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 @Suppress("CanBeParameter")
 class MatchedCharConsumer(

parser/src/main/kotlin/gay/pizza/pork/parser/PeekableSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/PeekableSource.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 interface PeekableSource<T> {
   val currentIndex: Int

parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndex.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndex.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 data class SourceIndex(val index: Int, val line: Int, val column: Int, val locationReliable: Boolean = true) {
   companion object {

parser/src/main/kotlin/gay/pizza/pork/parser/SourceIndexCharSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/SourceIndexCharSource.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 class SourceIndexCharSource(val delegate: CharSource) : CharSource by delegate {
   private var currentLineIndex = 1

parser/src/main/kotlin/gay/pizza/pork/parser/StringCharConsumer.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharConsumer.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 object StringCharConsumer : CharConsumer {
   override fun consume(type: TokenType, tokenizer: Tokenizer): String {

parser/src/main/kotlin/gay/pizza/pork/parser/StringCharSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/StringCharSource.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 class StringCharSource(
   val input: CharSequence,

parser/src/main/kotlin/gay/pizza/pork/parser/Token.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Token.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 class Token(val type: TokenType, val sourceIndex: SourceIndex, val text: String) {
   override fun toString(): String =

parser/src/main/kotlin/gay/pizza/pork/parser/TokenFamily.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenFamily.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 enum class TokenFamily : TokenTypeProperty {
   OperatorFamily,

parser/src/main/kotlin/gay/pizza/pork/parser/TokenSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 interface TokenSource : PeekableSource<Token> {
   fun peekTypeAhead(ahead: Int): TokenType

parser/src/main/kotlin/gay/pizza/pork/parser/TokenStream.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 class TokenStream(val tokens: List<Token>) {
   override fun toString(): String = tokens.toString()

parser/src/main/kotlin/gay/pizza/pork/parser/TokenStreamSource.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 class TokenStreamSource(val stream: TokenStream) : TokenSource {
   private var index = 0

parser/src/main/kotlin/gay/pizza/pork/parser/TokenType.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenType.kt

@@ -1,10 +1,10 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
-import gay.pizza.pork.parser.CharMatcher.*
-import gay.pizza.pork.parser.MatchedCharConsumer.Options.AllowEofTermination
-import gay.pizza.pork.parser.TokenTypeProperty.*
-import gay.pizza.pork.parser.TokenFamily.*
-import gay.pizza.pork.parser.TokenTypeProperty.AnyOf
+import gay.pizza.pork.tokenizer.CharMatcher.*
+import gay.pizza.pork.tokenizer.MatchedCharConsumer.Options.AllowEofTermination
+import gay.pizza.pork.tokenizer.TokenTypeProperty.*
+import gay.pizza.pork.tokenizer.TokenFamily.*
+import gay.pizza.pork.tokenizer.TokenTypeProperty.AnyOf
 
 enum class TokenType(vararg val properties: TokenTypeProperty) {
   NumberLiteral(NumericLiteralFamily, CharMatch(CharMatcher.AnyOf(

parser/src/main/kotlin/gay/pizza/pork/parser/TokenTypeProperty.kt → tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenTypeProperty.kt

@@ -1,4 +1,4 @@
-package gay.pizza.pork.parser
+package gay.pizza.pork.tokenizer
 
 interface TokenTypeProperty {
   class SingleChar(val char: Char) : TokenTypeProperty