tlang/tlang
tlang
/
tlang
mirror of https://github.com/tbklang/tlang.git
2
0
Fork 0
Code Issues 44 Projects 5 Releases Wiki Activity

Added new-line support 

Also added tracking for line number
This commit is contained in:
Tristan B. Velloza Kildaire 2021-03-03 11:41:16 +02:00
parent 45e707b582
commit 8d24f6ada2
1 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
source/tlang/compiler/lexer.d
View File

@ -30,6 +30,7 @@ public final class Lexer
{
/* The source to be lexed */
private string sourceCode;
private ulong line = 1;
/* The tokens */
private string[] tokens;
@ -59,8 +60,6 @@ public final class Lexer
/* Whether we are in a string "we are here" or not */
bool stringMode;
bool escapeMode;
while(position < sourceCode.length)
{
// gprintln("SrcCodeLen: "~to!(string)(sourceCode.length));
@ -96,6 +95,12 @@ public final class Lexer
splitterToken = "&&";
position += 2;
}
else if (currentChar == '\n')
{
line++;
position++;
}
else
{
splitterToken = ""~currentChar;
@ -112,10 +117,11 @@ public final class Lexer
currentToken = "";
}
/* Add the splitter token */
currentTokens ~= splitterToken;
gprintln("FInished process");
/* Add the splitter token (only if it isn't empty) */
if(splitterToken.length)
{
currentTokens ~= splitterToken;
}
}
else if(currentChar == '"')
{
@ -234,7 +240,8 @@ public final class Lexer
character == '+' || character == '-' || character == '/' ||
character == '%' || character == '*' || character == '&' ||
character == '{' || character == '}' || character == '=' ||
character == '|' || character == '^' || character == '!';
character == '|' || character == '^' || character == '!' ||
character == '\n';
}
/* Supported escapes \" */
@ -332,4 +339,17 @@ unittest
assert(currentLexer.getTokens() == ["'c'"]);
}
/* Test input: `2121\n2121` */
unittest
{
import std.algorithm.comparison;
string sourceCode = "2121\n2121";
Lexer currentLexer = new Lexer(sourceCode);
currentLexer.performLex();
gprintln("Collected "~to!(string)(currentLexer.getTokens()));
assert(currentLexer.getTokens() == ["2121", "2121"]);
}
/* TODO: Add more tests */