tlang/tlang
tlang
/
tlang
mirror of https://github.com/tbklang/tlang.git
2
0
Fork 0
Code Issues 44 Projects 5 Releases Wiki Activity

Fixed bug where we overshot array and added unittests

This commit is contained in:
Tristan B. Velloza Kildaire 2021-03-02 23:22:42 +02:00
parent 1b0de5776b
commit f23d884b90
1 changed files with 94 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

97
source/tlang/commandline/lexer.d
View File

@ -1,5 +1,60 @@
module compiler.lexer;
import std.container.slist;
import gogga;
import std.conv : to;
/* Test input: `hello "world";` */
unittest
{
import std.algorithm.comparison;
string sourceCode = "hello \"world\";";
Lexer currentLexer = new Lexer(sourceCode);
currentLexer.performLex();
gprintln("Collected "~to!(string)(currentLexer.getTokens()));
assert(currentLexer.getTokens() == ["hello", "\"world\"",";"]);
}
/* Test input: `hello "world"|| ` */
unittest
{
import std.algorithm.comparison;
string sourceCode = "hello \"world\"|| ";
Lexer currentLexer = new Lexer(sourceCode);
currentLexer.performLex();
gprintln("Collected "~to!(string)(currentLexer.getTokens()));
assert(currentLexer.getTokens() == ["hello", "\"world\"","||"]);
}
/* Test input: `hello "world"||` */
unittest
{
import std.algorithm.comparison;
string sourceCode = "hello \"world\"||";
Lexer currentLexer = new Lexer(sourceCode);
currentLexer.performLex();
gprintln("Collected "~to!(string)(currentLexer.getTokens()));
assert(currentLexer.getTokens() == ["hello", "\"world\"","||"]);
}
/* Test input: `hello "world";` */
unittest
{
import std.algorithm.comparison;
string sourceCode = "hello \"world\";";
Lexer currentLexer = new Lexer(sourceCode);
currentLexer.performLex();
gprintln("Collected "~to!(string)(currentLexer.getTokens()));
assert(currentLexer.getTokens() == ["hello", "\"world\"",";"]);
}
// string sourceCode = "hello \"world\"|| ";
//string sourceCode = "hello \"world\"||"; /* TODO: Implement this one */
// string sourceCode = "hello;";
public final class Lexer
{
/* The source to be lexed */
@ -16,6 +71,15 @@ public final class Lexer
/* Perform the lexing process */
public void performLex()
{
// SList!(string) tokenThing;
// tokenThing.insert("1");
// tokenThing.insert("2");
// import std.stdio;
// writeln(tokenThing.front());
// writeln(tokenThing.front());
string[] currentTokens;
string currentToken;
ulong position;
@ -23,8 +87,11 @@ public final class Lexer
bool stringMode;
while(position != sourceCode.length)
while(position < sourceCode.length)
{
// gprintln("SrcCodeLen: "~to!(string)(sourceCode.length));
// gprintln("Position: "~to!(string)(position));
currentChar = sourceCode[position];
if(currentChar == ' ' && !stringMode)
@ -40,6 +107,30 @@ public final class Lexer
}
else if(isSpliter(currentChar) && !stringMode)
{
/* The splitter token to finally insert */
string splitterToken;
/* Check if we need to do combinators (e.g. for ||, &&) */
/* TODO: Second operand in condition out of bounds */
if(currentChar == '|' && sourceCode[position+1] == '|')
{
splitterToken = "||";
position += 2;
}
else if(currentChar == '&' && sourceCode[position+1] == '&')
{
splitterToken = "&&";
position += 2;
}
else
{
splitterToken = ""~currentChar;
position++;
}
/* Flush the current token (if one exists) */
if(currentToken.length)
{
@ -48,9 +139,9 @@ public final class Lexer
}
/* Add the splitter token */
currentTokens ~= ""~currentChar;
currentTokens ~= splitterToken;
position++;
gprintln("FInished process");
}
else if(currentChar == '"')
{