diff --git a/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt b/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt index d4e70d0..c7d818d 100644 --- a/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt +++ b/minimal/src/main/kotlin/gay/pizza/pork/minimal/Tool.kt @@ -29,8 +29,8 @@ abstract class Tool { fun parse(attribution: NodeAttribution = DiscardNodeAttribution): CompilationUnit = Parser(tokenize(), attribution).parseCompilationUnit() - fun highlight(scheme: HighlightScheme): List = - Highlighter(scheme).highlight(tokenize().stream()) + fun highlight(scheme: HighlightScheme): Sequence = + Highlighter(scheme).highlight(tokenize()) fun reprint(): String = buildString { visit(Printer(this)) } @@ -44,6 +44,13 @@ abstract class Tool { return resultingScope.value("main") as CallableFunction } + fun loadMainFunctionStandard(scope: Scope, quiet: Boolean = false): CallableFunction = + loadMainFunction(scope, setupEvaluator = { + addNativeProvider("internal", InternalNativeProvider(quiet = quiet)) + addNativeProvider("ffi", FfiNativeProvider()) + addNativeProvider("java", JavaNativeProvider()) + }) + fun buildWorld(): World { val fileContentSource = createContentSource() val dynamicImportSource = DynamicImportSource() @@ -54,11 +61,7 @@ abstract class Tool { } fun run(scope: Scope, quiet: Boolean = false) { - val main = loadMainFunction(scope, setupEvaluator = { - addNativeProvider("internal", InternalNativeProvider(quiet = quiet)) - addNativeProvider("ffi", FfiNativeProvider()) - addNativeProvider("java", JavaNativeProvider()) - }) + val main = loadMainFunctionStandard(scope, quiet = quiet) main.call(emptyList(), CallStack()) } } diff --git a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt index a2a51bf..77be26e 100644 --- a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/Highlighter.kt @@ -1,6 +1,6 @@ package gay.pizza.pork.tokenizer class Highlighter(val scheme: HighlightScheme) { - fun highlight(stream: TokenStream): List = - stream.tokens.map { scheme.highlight(it) } + fun highlight(source: TokenSource): Sequence = + source.sequence().map { scheme.highlight(it) } } diff --git a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ListTokenSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ListTokenSource.kt new file mode 100644 index 0000000..58c683e --- /dev/null +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/ListTokenSource.kt @@ -0,0 +1,32 @@ +package gay.pizza.pork.tokenizer + +class ListTokenSource(val tokens: List) : TokenSource { + private var index = 0 + + override val currentIndex: Int + get() = index + + override fun next(): Token { + if (index == tokens.size) { + return tokens.last() + } + val char = tokens[index] + index++ + return char + } + + override fun peek(): Token { + if (index == tokens.size) { + return tokens.last() + } + return tokens[index] + } + + override fun peekTypeAhead(ahead: Int): TokenType { + val calculated = index + ahead + if (calculated >= tokens.size) { + return tokens.last().type + } + return tokens[calculated].type + } +} diff --git a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt index 783d71d..4506df0 100644 --- a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt +++ b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenSource.kt @@ -15,5 +15,17 @@ interface TokenSource : PeekableSource { return tokens } - fun stream(): TokenStream = TokenStream(consumeAllRemainingTokens()) + fun sequence(): Sequence { + var endFlag = false + return generateSequence { + if (endFlag) { + return@generateSequence null + } + val token = next() + if (token.type == TokenType.EndOfFile) { + endFlag = true + token + } else token + } + } } diff --git a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt deleted file mode 100644 index 5cad828..0000000 --- a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStream.kt +++ /dev/null @@ -1,5 +0,0 @@ -package gay.pizza.pork.tokenizer - -class TokenStream(val tokens: List) { - override fun toString(): String = tokens.toString() -} diff --git a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt b/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt deleted file mode 100644 index 4407bc0..0000000 --- a/tokenizer/src/main/kotlin/gay/pizza/pork/tokenizer/TokenStreamSource.kt +++ /dev/null @@ -1,32 +0,0 @@ -package gay.pizza.pork.tokenizer - -class TokenStreamSource(val stream: TokenStream) : TokenSource { - private var index = 0 - - override val currentIndex: Int - get() = index - - override fun next(): Token { - if (index == stream.tokens.size) { - return stream.tokens.last() - } - val char = stream.tokens[index] - index++ - return char - } - - override fun peek(): Token { - if (index == stream.tokens.size) { - return stream.tokens.last() - } - return stream.tokens[index] - } - - override fun peekTypeAhead(ahead: Int): TokenType { - val calculated = index + ahead - if (calculated >= stream.tokens.size) { - return stream.tokens.last().type - } - return stream.tokens[calculated].type - } -} diff --git a/tool/src/main/kotlin/gay/pizza/pork/tool/RunCommand.kt b/tool/src/main/kotlin/gay/pizza/pork/tool/RunCommand.kt index d78d594..70777e9 100644 --- a/tool/src/main/kotlin/gay/pizza/pork/tool/RunCommand.kt +++ b/tool/src/main/kotlin/gay/pizza/pork/tool/RunCommand.kt @@ -20,11 +20,7 @@ class RunCommand : CliktCommand(help = "Run Program", name = "run") { override fun run() { val tool = FileTool(PlatformFsProvider.resolve(path)) val scope = Scope.root() - val main = tool.loadMainFunction(scope, setupEvaluator = { - addNativeProvider("internal", InternalNativeProvider(quiet = quiet)) - addNativeProvider("ffi", FfiNativeProvider()) - addNativeProvider("java", JavaNativeProvider()) - }) + val main = tool.loadMainFunctionStandard(scope, quiet = quiet) if (dumpScope) { val functionContext = main as FunctionContext