#ifndef INCLUDE_LEXER_H
#define INCLUDE_LEXER_H
#print "[include <lexer.h>]\n"
#ifndef INCLUDE_STRING_H
#include "../lib/strings.h"
#endif
#ifndef INCLUDE_MEM_H
#include "../lib/mem.h"
#endif
/** Splits text into tokens
* Author :Pavel Yakovlev
* Homepage:https://vk.com/pavelyakov39
* Ver. : 1.51
*/
/** Example:
* lexer lex;
* lex.load("var a=123;");
* lex.next();
* lex.token; //TOKEN == 'var'
* lex.type ; //TYPE == LEX_VAR
*
* lex.next();
* lex.token; //TOKEN == 'a'
* lex.type ; //TYPE == LEX_VAR
*
* lex.next();
* lex.token; //TOKEN == '='
* lex.type ; //TYPE == LEX_IND
*
* lex.next();
* lex.token; //TOKEN == '123'
* lex.type ; //TYPE == LEX_DEC
*
* lex.next();
* lex.token; //TOKEN == ';'
* lex.type ; //TYPE == LEX_IND
*
* lex.next();
* lex.token; //TOKEN == ''
* lex.type ; //TYPE == LEX_END
*/
#define LEX_END 1
#define LEX_STR 2
#define LEX_DEC 3
#define LEX_VAR 4
#define LEX_FNC 5
#define LEX_IND 6
#define LEX_NUL 0
:char const_token_lexer[1024];
:struct lexer
{
byte cmd;
dword token,text,mem_list,count,buffer_loading;
dword str_buffer;
byte type;
char quote;
signed length;
dword next(void);
dword back(void);
dword list(void);
void free(void);
dword position(dword __);
void load(dword _text);
void expected(dword _text);
};
:dword back(void)
{
}
:dword lexer::list(void)
{
dword count_mem,buf_loop,pos;
count_mem = 0;
buf_loop = 5000; // на тыс элементов.
count = 0;
buffer_loading = malloc(buf_loop);
pos = buffer_loading;
while(type!=LEX_END)
{
pos+=count_mem;
next();
DSDWORD[pos] = strndup(token,length);
pos+=4;
DSBYTE [pos] = type;
pos++;
count++;
if(pos-buffer_loading>buf_loop)
{
buf_loop*=2;
buffer_loading = realloc(buffer_loading,buf_loop);
}
}
return buffer_loading;
}
:void lexer::free(void)
{
dword z;
z = count;
while(z)
{
z--;
position(z);
::free(token);
}
count = 0;
::free(buffer_loading);
}
:dword lexer::position(dword __)
{
dword pos1;
if(!count)list();
if(__>=count)__=count-1;
else if(__<0)__=0;
pos1 = __*5;
pos1 += buffer_loading;
token = DSDWORD[pos1];
pos1++;
type = DSBYTE[pos1];
return token;
}
:void lexer::expected(dword _text)
{
notify(_text);
ExitProcess();
}
:void lexer::load(dword _text)
{
text = _text;
count = 0;
str_buffer = 0;
}
:dword lexer::next(void)
{
char s;
dword len_str_buf,tmp;
dword pos,in;
pos = #const_token_lexer;
in = text;
//len_str_buf = 1024;
if(str_buffer)::free(str_buffer);
NEXT_TOKEN:
length = 0;
loop()
{
s = DSBYTE[in];
if(s!=9)&&(s!=10)&&(s!=13)&&(s!=32)break;
in++;
text++;
}
if(s==0){type=LEX_END;DSBYTE[pos]=0;token="";return token;}
if(s=='/')
{
in++;
s = DSBYTE[in];
// Line comments
if(s=='/')
{
loop()
{
in++;
s = DSBYTE[in];
if(s==10)||(s==13)||(s==0)goto NEXT_TOKEN;
/* Add comments*/
}
}
if(s=='*')
{
loop()
{
in++;
s = DSBYTE[in];
if(s=='*')if(DSBYTE[in+1]=='/')
{
in+=2;
goto NEXT_TOKEN;
}
}
}
}
if (strchr("=<>!~&|#",s))
{
loop()
{
if (!strchr("=<>!~&|#",s)) break;
DSBYTE[pos] = s;
pos++;
in++;
s = DSBYTE[in];
}
type = LEX_IND;
}
else if (strchr(";(,)}{[]+-.*/:^%?$@№`",s))
{
DSBYTE[pos] = s;
pos++;
type = LEX_IND;
in++;
}
else if(s>='0')&&(s<='9')
{
loop()
{
if(s<'0')||(s>'9')if(s!='.')break;
DSBYTE[pos] = s;
pos++;
in++;
s = DSBYTE[in];
}
type = LEX_DEC;
}
else if(s>='A')&&(s<='z')&&(!strchr("[]\\^`",s))
{
loop()
{
if(s<'A')||(s>'z')if(s<'0')||(s>'9')break;
if(strchr("[]\\^`",s))break;
DSBYTE[pos] = s;
pos++;
in++;
s = DSBYTE[in];
}
loop()
{
s = DSBYTE[in];
if(s!=9)if(s!=10)if(s!=13)if(s!=32)break;
in++;
text++;
}
type = LEX_VAR;
if(s=='(')type = LEX_FNC;
}
else if(s=='"')||(s=='\'')
{
quote = s;
in++;
tmp = in;
s = DSBYTE[in];
loop()
{
if(s=='\\')
{
in++;
s = DSBYTE[in];
if(!s){type = LEX_STR;goto GOTO_LEX_END;}
if(!cmd)switch(s)
{
case 'n':s='\n';break;
case 'r':s='\r';break;
case 't':s='\t';break;
}
else {
DSBYTE[pos] = '\\';
pos++;
}
goto LEX_STEP_1;
}
if(!s){type = LEX_STR;goto GOTO_LEX_END;}
else if(s==quote)break;
LEX_STEP_1:
DSBYTE[pos] = s;
pos++;
in++;
/*if(in-tmp>len_str_buf)
{
if(str_buffer)
{
tmp = len_str_buf;
len_str_buf+=1024;
str_buffer = realloc(str_buffer,len_str_buf+1);
strlcpy(str_buffer+tmp,#const_token_lexer,1024);
pos = #const_token_lexer;
}
else {
len_str_buf+=1024;
str_buffer = malloc(len_str_buf+1);
strlcpy(str_buffer,#const_token_lexer,1024);
pos = #const_token_lexer;
}
}*/
s = DSBYTE[in];
}
in++;
/*tmp = pos-in;
if(str_buffer)
{
if(tmp)
{
str_buffer = realloc(str_buffer,tmp+1);
strlcpy(str_buffer+len_str_buf,#const_token_lexer,tmp);
}
type = LEX_STR;
length = len_str_buf+tmp;
text = in;
tmp = str_buffer+length;
DSBYTE[tmp] = 0;
token = str_buffer;
return token;
}*/
type = LEX_STR;
}
else {
in++;
type = LEX_NUL;
DSBYTE[pos] = s;
pos++;
}
GOTO_LEX_END:
length = in-text;
text = in;
DSBYTE[pos] = 0;
token = #const_token_lexer;
return token;
}
#endif