Rev 5631 | Go to most recent revision | Details | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
5598 | pavelyakov | 1 | #ifndef INCLUDE_LEXER_H |
2 | #define INCLUDE_LEXER_H |
||
3 | |||
4 | #ifndef INCLUDE_STRING_H |
||
5 | #include "../lib/strings.h" |
||
6 | #endif |
||
7 | /** Splits text into tokens |
||
8 | * Author :Pavel Yakovlev |
||
9 | * Homepage:https://vk.com/pavelyakov39 |
||
10 | */ |
||
11 | |||
12 | /** Example: |
||
13 | * lexer lex; |
||
14 | * lex.load("var a=123;"); |
||
15 | * lex.next(); |
||
16 | * lex.token; //TOKEN == 'var' |
||
17 | * lex.type ; //TYPE == LEX_VAR |
||
18 | * |
||
19 | * lex.next(); |
||
20 | * lex.token; //TOKEN == 'a' |
||
21 | * lex.type ; //TYPE == LEX_VAR |
||
22 | * |
||
23 | * lex.next(); |
||
24 | * lex.token; //TOKEN == '=' |
||
25 | * lex.type ; //TYPE == LEX_IND |
||
26 | * |
||
27 | * lex.next(); |
||
28 | * lex.token; //TOKEN == '123' |
||
29 | * lex.type ; //TYPE == LEX_DEC |
||
30 | * |
||
31 | * lex.next(); |
||
32 | * lex.token; //TOKEN == ';' |
||
33 | * lex.type ; //TYPE == LEX_IND |
||
34 | * |
||
35 | * lex.next(); |
||
36 | * lex.token; //TOKEN == '' |
||
37 | * lex.type ; //TYPE == LEX_END |
||
38 | */ |
||
39 | |||
40 | #define LEX_END 1 |
||
41 | #define LEX_STR 2 |
||
42 | #define LEX_DEC 3 |
||
43 | #define LEX_VAR 4 |
||
44 | #define LEX_FNC 5 |
||
45 | #define LEX_IND 6 |
||
46 | #define LEX_NUL 0 |
||
47 | |||
48 | :char const_token_lexer[1024]; |
||
49 | :struct lexer |
||
50 | { |
||
51 | byte cmd; |
||
52 | dword token,text; |
||
53 | byte type; |
||
54 | char quote; |
||
55 | signed count,length; |
||
56 | dword next(void); |
||
57 | dword back(void); |
||
58 | void load(dword _text); |
||
59 | void expected(dword _text); |
||
60 | }; |
||
61 | |||
62 | |||
63 | :void expected(dword _text) |
||
64 | { |
||
65 | notify(_text); |
||
66 | ExitProcess(); |
||
67 | } |
||
68 | |||
69 | :void lexer::load(dword _text) |
||
70 | { |
||
71 | text = _text; |
||
72 | } |
||
73 | |||
74 | :dword lexer::next(void) |
||
75 | { |
||
76 | char s; |
||
77 | dword pos,in; |
||
78 | pos = #const_token_lexer; |
||
79 | in = text; |
||
80 | |||
81 | NEXT_TOKEN: |
||
82 | length = 0; |
||
83 | loop() |
||
84 | { |
||
85 | s = DSBYTE[in]; |
||
86 | if(s!=9)&&(s!=10)&&(s!=13)&&(s!=32)break; |
||
87 | in++; |
||
88 | text++; |
||
89 | } |
||
90 | |||
91 | if(s==0){type=LEX_END;DSBYTE[pos]=0;token="";return token;} |
||
92 | |||
93 | if(s=='/') |
||
94 | { |
||
95 | in++; |
||
96 | s = DSBYTE[in]; |
||
97 | |||
98 | // Line comments |
||
99 | if(s=='/') |
||
100 | { |
||
101 | loop() |
||
102 | { |
||
103 | in++; |
||
104 | s = DSBYTE[in]; |
||
105 | if(s==10)||(s==13)||(s==0)goto NEXT_TOKEN; |
||
106 | /* Add comments*/ |
||
107 | } |
||
108 | } |
||
109 | if(s=='*') |
||
110 | { |
||
111 | loop() |
||
112 | { |
||
113 | in++; |
||
114 | s = DSBYTE[in]; |
||
115 | if(s=='*')if(DSBYTE[in+1]=='/') |
||
116 | { |
||
117 | in+=2; |
||
118 | goto NEXT_TOKEN; |
||
119 | } |
||
120 | } |
||
121 | } |
||
122 | } |
||
123 | |||
124 | if (strchr("=<>!~&|#",s)) |
||
125 | { |
||
126 | loop() |
||
127 | { |
||
128 | if (!strchr("=<>!~&|#",s)) break; |
||
129 | |||
130 | DSBYTE[pos] = s; |
||
131 | pos++; |
||
132 | |||
133 | in++; |
||
134 | s = DSBYTE[in]; |
||
135 | } |
||
136 | type = LEX_IND; |
||
137 | } |
||
138 | else if (strchr(";(,)}{[]+-.*/:^%?$@№`",s)) |
||
139 | { |
||
140 | DSBYTE[pos] = s; |
||
141 | pos++; |
||
142 | type = LEX_IND; |
||
143 | in++; |
||
144 | } |
||
145 | else if(s>='0')&&(s<='9') |
||
146 | { |
||
147 | loop() |
||
148 | { |
||
149 | if(s<'0')||(s>'9')if(s!='.')break; |
||
150 | |||
151 | DSBYTE[pos] = s; |
||
152 | pos++; |
||
153 | |||
154 | in++; |
||
155 | s = DSBYTE[in]; |
||
156 | } |
||
157 | type = LEX_DEC; |
||
158 | } |
||
159 | else if(s>='A')&&(s<='z')&&(!strchr("[]\\^`",s)) |
||
160 | { |
||
161 | loop() |
||
162 | { |
||
163 | if(s<'A')||(s>'z')if(s<'0')||(s>'9')break; |
||
164 | if(strchr("[]\\^`",s))break; |
||
165 | |||
166 | DSBYTE[pos] = s; |
||
167 | pos++; |
||
168 | |||
169 | in++; |
||
170 | s = DSBYTE[in]; |
||
171 | } |
||
172 | |||
173 | loop() |
||
174 | { |
||
175 | s = DSBYTE[in]; |
||
176 | if(s!=9)if(s!=10)if(s!=13)if(s!=32)break; |
||
177 | in++; |
||
178 | text++; |
||
179 | } |
||
180 | type = LEX_VAR; |
||
181 | if(s=='(')type = LEX_FNC; |
||
182 | } |
||
183 | else if(s=='"')||(s=='\'') |
||
184 | { |
||
185 | quote = s; |
||
186 | in++; |
||
187 | s = DSBYTE[in]; |
||
188 | loop() |
||
189 | { |
||
190 | if(s=='\\') |
||
191 | { |
||
192 | in++; |
||
193 | s = DSBYTE[in]; |
||
194 | if(!s){type = LEX_STR;goto GOTO_LEX_END;} |
||
195 | if(!cmd)switch(s) |
||
196 | { |
||
197 | case 'n':s='\n';break; |
||
198 | case 'r':s='\r';break; |
||
199 | case 't':s='\t';break; |
||
200 | } |
||
201 | else { |
||
202 | DSBYTE[pos] = '\\'; |
||
203 | pos++; |
||
204 | } |
||
205 | goto LEX_STEP_1; |
||
206 | } |
||
207 | if(!s){type = LEX_STR;goto GOTO_LEX_END;} |
||
208 | else if(s==quote)break; |
||
209 | LEX_STEP_1: |
||
210 | DSBYTE[pos] = s; |
||
211 | pos++; |
||
212 | in++; |
||
213 | s = DSBYTE[in]; |
||
214 | } |
||
215 | in++; |
||
216 | type = LEX_STR; |
||
217 | } |
||
218 | else { |
||
219 | in++; |
||
220 | type = LEX_NUL; |
||
221 | DSBYTE[pos] = s; |
||
222 | pos++; |
||
223 | } |
||
224 | GOTO_LEX_END: |
||
225 | length = in-text; |
||
226 | text = in; |
||
227 | DSBYTE[pos] = 0; |
||
228 | token = #const_token_lexer; |
||
229 | return token; |
||
230 | } |
||
231 | |||
232 | #endif'0')||(s>'A')||(s>='z')&&(!strchr("[]\\^`",s)) |