slang_frontend/
token.rs

1use std::fmt::Display;
2
3/// Types of tokens in the language lexer
4#[derive(Debug, PartialEq)]
5pub enum Tokentype {
6    Identifier,     // x, y, myVar
7    IntegerLiteral, // 123
8    FloatLiteral,   // 123.45
9    StringLiteral,  // "hello world"
10    BooleanLiteral, // true, false
11    Let,            // let
12    Mut,            // mut
13    Plus,           // +
14    Minus,          // -
15    Multiply,       // *
16    Divide,         // /
17    Not,            // !
18    And,            // &&
19    Or,             // ||
20    Greater,        // >
21    Less,           // <
22    GreaterEqual,   // >=
23    LessEqual,      // <=
24    EqualEqual,     // ==
25    NotEqual,       // !=
26    Invalid,        // Unrecognized token
27    Equal,          // =
28    Colon,          // :
29    Semicolon,      // ;
30    Struct,         // struct
31    LeftBrace,      // {
32    RightBrace,     // }
33    Comma,          // ,
34    Fn,             // fn
35    LeftParen,      // (
36    RightParen,     // )
37    Arrow,          // ->
38    Return,         // return
39    If,             // if
40    Else,           // else
41
42    Eof, // End of file
43}
44
45impl Display for Tokentype {
46    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
47        write!(
48            f,
49            "{}",
50            match self {
51                Tokentype::Identifier => "identifier",
52                Tokentype::IntegerLiteral => "integer literal",
53                Tokentype::FloatLiteral => "float literal",
54                Tokentype::StringLiteral => "string literal",
55                Tokentype::BooleanLiteral => "boolean literal",
56                Tokentype::Let => "let keyword",
57                Tokentype::Mut => "mut keyword",
58                Tokentype::Plus => "'+'",
59                Tokentype::Minus => "'-'",
60                Tokentype::Multiply => "'*'",
61                Tokentype::Divide => "'/'",
62                Tokentype::Not => "'!'",
63                Tokentype::And => "'&&'",
64                Tokentype::Or => "'||'",
65                Tokentype::Greater => "'>'",
66                Tokentype::Less => "'<'",
67                Tokentype::GreaterEqual => "'>='",
68                Tokentype::LessEqual => "'<='",
69                Tokentype::EqualEqual => "'=='",
70                Tokentype::NotEqual => "'!='",
71                Tokentype::Invalid => "invalid token",
72                Tokentype::Equal => "'='",
73                Tokentype::Colon => "':'",
74                Tokentype::Semicolon => "';'",
75                Tokentype::Struct => "sturct keyword",
76                Tokentype::LeftBrace => "'{'",
77                Tokentype::RightBrace => "'}'",
78                Tokentype::Comma => "','",
79                Tokentype::Fn => "fn keyword",
80                Tokentype::LeftParen => "'('",
81                Tokentype::RightParen => "')'",
82                Tokentype::Arrow => "'->'",
83                Tokentype::Return => "return keyword",
84                Tokentype::If => "if keyword",
85                Tokentype::Else => "else keyword",
86                Tokentype::Eof => "<EOF>",
87            }
88        )
89    }
90}
91
92/// Represents a token in the source code
93pub struct Token {
94    /// The type of the token
95    pub token_type: Tokentype,
96    /// The actual text of the token
97    pub lexeme: String,
98    /// Position index - used with LineInfo to determine line number
99    pub pos: usize,
100}
101
102impl Token {
103    /// Creates a new token with the given type, lexeme, and position
104    pub fn new(token_type: Tokentype, lexeme: String, pos: usize) -> Token {
105        Token {
106            token_type,
107            lexeme,
108            pos,
109        }
110    }
111}
112
113impl Display for Token {
114    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
115        write!(f, "{}: {}", self.token_type, self.lexeme)
116    }
117}
slang_frontend/token.rs

slang_frontend/
token.rs
