/*
* Tiny BASIC
* Tokenisation module
*
* Copyright (C) Damian Gareth Walker 2019
* Created: 04-Aug-2019
*/
/* included headers */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "token.h"
#include "tokeniser.h"
#include "common.h"
/*
* Data definitions
*/
/* modes of reading */
typedef enum {
DEFAULT_MODE, /* we have no idea what's coming */
COMMENT_MODE, /* reading a comment */
WORD_MODE, /* reading an identifier or keyword */
NUMBER_MODE, /* reading a numeric constant */
LESS_THAN_MODE, /* reading an operator staring with < */
GREATER_THAN_MODE, /* reading an operator starting with > */
STRING_LITERAL_MODE, /* reading a string literal */
UNKNOWN_MODE /* we are lost */
} Mode;
/* current state information */
typedef struct {
Token *token; /* token to return */
Mode mode; /* current reading mode */
int ch; /* last-read character */
char *content; /* content of token under construction */
int max; /* memory reserved for content */
} TokeniserState;
/* Private data */
typedef struct {
FILE *input; /* the input file */
int line, /* current line in the input file */
pos, /* current position on the input line */
start_line, /* line on which a token started */
start_pos; /* position on which a token started */
} Private;
/*
* File level variables
*/
/* convenience variables */
static TokenStream *this; /* token stream passed in to public method */
static Private *data; /* private data for this */
/*
* Level 2 Tokeniser Routines
*/
/*
* Read a character and update the position counter
* globals:
* int line current line after character read
* int pos current character position after character read
* params:
* TokeniserState* state current state of the tokeniser
* returns:
* int character just read
*/
static int read_character (TokeniserState *state) {
int ch; /* character read from stream */
/* read the character */
ch
= fgetc (data
->input
);
/* update the position and line counters */
if (ch == '\n') {
++data->line;
data->pos = 0;
} else {
++data->pos;
}
/* return the character */
return ch;
}
/*
* Push a character back into the input stream and update position markers
* globals:
* int line line number rolled back
* int pos character position rolled back
* params:
* TokeniserState* state current state of the tokeniser
*/
static void unread_character (TokeniserState *state) {
ungetc (state
->ch
, data
->input
); if (state->ch == '\n')
--data->line;
else
--data->pos;
}
/*
* Append the last read character to the token content
* params:
* TokeniserState* state current state of the tokeniser
*/
static void store_character (TokeniserState *state) {
/* variable declarations */
char *temp; /* temporary pointer to content */
int length; /* current length of token */
/* allocate more memory for the token content if necessary */
if (strlen (state
->content
) == state
->max
- 1) { temp = state->content;
state->max *= 2;
state
->content
= malloc (state
->max
); strcpy (state
->content
, temp
); }
/* now add the character to the token */
length
= strlen (state
->content
); state->content [length++] = state->ch;
state->content [length] = '\0';
}
/*
* Identify the various recognised symbols
* params:
* int ch the character to identify
* returns:
* TokenClass the token class recognised by the parser
*/
static TokenClass identify_symbol (int ch) {
switch (ch) {
case '+':
return TOKEN_PLUS;
break;
case '-':
return TOKEN_MINUS;
break;
case '*':
return TOKEN_MULTIPLY;
break;
case '/':
return TOKEN_DIVIDE;
break;
case '=':
return TOKEN_EQUAL;
break;
case '(':
return TOKEN_LEFT_PARENTHESIS;
break;
case ')':
return TOKEN_RIGHT_PARENTHESIS;
break;
case ',':
return TOKEN_COMMA;
break;
default:
return TOKEN_SYMBOL;
}
}
static TokenClass identify_word (char *word) {
return TOKEN_VARIABLE;
else if (! tinybasic_strcmp (word, "LET"))
return TOKEN_LET;
else if (! tinybasic_strcmp (word, "IF"))
return TOKEN_IF;
else if (! tinybasic_strcmp (word, "THEN"))
return TOKEN_THEN;
else if (! tinybasic_strcmp (word, "GOTO"))
return TOKEN_GOTO;
else if (! tinybasic_strcmp (word, "GOSUB"))
return TOKEN_GOSUB;
else if (! tinybasic_strcmp (word, "RETURN"))
return TOKEN_RETURN;
else if (! tinybasic_strcmp (word, "END"))
return TOKEN_END;
else if (! tinybasic_strcmp (word, "PRINT"))
return TOKEN_PRINT;
else if (! tinybasic_strcmp (word, "INPUT"))
return TOKEN_INPUT;
else if (! tinybasic_strcmp (word, "REM"))
return TOKEN_REM;
else
return TOKEN_WORD;
}
/*
* Identify compound (multi-character) symbols.
* Also identifies some single-character symbols that can form
* the start of multi-character symbols.
* params:
* char* symbol the symbol to identify
* returns:
* TokenClass the identification
*/
static TokenClass identify_compound_symbol (char *symbol) {
return TOKEN_UNEQUAL;
else if (! strcmp (symbol
, "<")) return TOKEN_LESSTHAN;
else if (! strcmp (symbol
, "<=")) return TOKEN_LESSOREQUAL;
else if (! strcmp (symbol
, ">")) return TOKEN_GREATERTHAN;
else if (! strcmp (symbol
, ">=")) return TOKEN_GREATEROREQUAL;
else
return TOKEN_SYMBOL;
}
/*
* Level 1 Tokeniser Routines
*/
/*
* Default mode - deal with character when state is unknown
* globals:
* int line current line in the source file
* int pos current character position in the source
* int start_line line on which the current token started
* int start_pos char pos on which the current token started
* params:
* TokeniserState* state current state of the tokeniser
*/
static void default_mode (TokeniserState *state) {
/* deal with non-EOL whitespace */
if (state->ch == ' ' ||
state->ch == '\t') {
state->ch = read_character (state);
data->start_line = data->line;
data->start_pos = data->pos;
}
/* deal with EOL whitespace */
else if (state->ch == '\n') {
data->start_line = data->line - 1;
data->start_pos = data->pos;
state->token = new_Token_init
(TOKEN_EOL, data->start_line, data->start_pos, state->content);
}
/* alphabetic characters start a word */
else if ((state->ch >= 'A' && state->ch <= 'Z') ||
(state->ch >= 'a' && state->ch <= 'z')) {
data->start_line = data->line;
data->start_pos = data->pos;
state->mode = WORD_MODE;
}
/* digits start a number */
else if (state->ch >= '0' && state->ch <= '9')
state->mode = NUMBER_MODE;
/* check for tokens starting with less-than (<, <=, <>) */
else if (state->ch == '<') {
data->start_line = data->line;
data->start_pos = data->pos;
store_character (state);
state->ch = read_character (state);
state->mode = LESS_THAN_MODE;
}
/* check for tokens starting with greater-than (>, >=) */
else if (state->ch == '>') {
data->start_line = data->line;
data->start_pos = data->pos;
store_character (state);
state->ch = read_character (state);
state->mode = GREATER_THAN_MODE;
}
/* deal with other symbol operators */
else if (strchr ("+-*/=(),", state
->ch
) != NULL
) { data->start_line = data->line;
data->start_pos = data->pos;
store_character (state);
state->token = new_Token_init (identify_symbol (state->ch),
data->start_line, data->start_pos, state->content);
}
/* double quotes start a string literal */
else if (state->ch == '"') {
data->start_line = data->line;
data->start_pos = data->pos;
state->ch = read_character (state);
state->mode = STRING_LITERAL_MODE;
}
/* detect end of file */
else if (state->ch == EOF) {
data->start_line = data->line;
data->start_pos = data->pos;
state->token = new_Token_init
(TOKEN_EOF, data->start_line, data->start_pos, state->content);
}
/* other characters are illegal */
else {
data->start_line = data->line;
data->start_pos = data->pos;
store_character (state);
state->token = new_Token_init
(TOKEN_ILLEGAL, data->start_line, data->start_pos, state->content);
}
}
/*
* Word mode - deal with character when building a word token
* globals:
* int start_line line on which the current token started
* int start_pos char pos on which the current token started
* params:
* TokeniserState* state current state of the tokeniser
*/
static void word_mode (TokeniserState *state) {
/* local variables */
TokenClass class; /* recognised class of keyword */
/* add letters and digits to the token */
if ((state->ch >= 'A' && state->ch <= 'Z') ||
(state->ch >= 'a' && state->ch <= 'z')) {
store_character (state);
state->ch = read_character (state);
}
/* other characters are pushed back for the next token */
else {
if (state->ch != EOF)
unread_character (state);
class = identify_word (state->content);
if (class == TOKEN_REM) {
*state->content = '\0';
state->mode = COMMENT_MODE;
}
else
state->token = new_Token_init
(class, data->start_line, data->start_pos, state->content);
}
}
/*
* Comment mode - skip till end of line after a REM
* globals:
* int start_line line on which the current token started
* int start_pos char pos on which the current token started
* params:
* TokeniserState* state current state of the tokeniser
*/
static void comment_mode (TokeniserState *state) {
if (state->ch == '\n')
state->mode = DEFAULT_MODE;
else
state->ch = read_character (state);
}
/*
* Number mode - building a number token (integer only)
* globals:
* int start_line line on which the current token started
* int start_pos char pos on which the current token started
* params:
* TokeniserState* state current state of the tokeniser
*/
static void number_mode (TokeniserState *state) {
/* add digits to the token */
if (state->ch >= '0' && state->ch <= '9') {
store_character (state);
state->ch = read_character (state);
}
/* other characters are pushed back for the next token */
else {
if (state->ch != EOF)
unread_character (state);
state->token = new_Token_init
(TOKEN_NUMBER, data->start_line, data->start_pos, state->content);
}
}
/*
* Less than mode - checking for <> and <= operators
* globals:
* int start_line line on which the current token started
* int start_pos char pos on which the current token started
* params:
* TokeniserState* state current state of the tokeniser
*/
static void less_than_mode (TokeniserState *state) {
if (state->ch == '=' || state->ch == '>')
store_character (state);
else
unread_character (state);
state->token = new_Token_init
(identify_compound_symbol (state->content), data->start_line,
data->start_pos, state->content);
}
/*
* Greater than mode - checking for >= and >< operators
* globals:
* int start_line line on which the current token started
* int start_pos char pos on which the current token started
* params:
* TokeniserState* state current state of the tokeniser
*/
static void greater_than_mode (TokeniserState *state) {
if (state->ch == '=' || state->ch == '<')
store_character (state);
else
ungetc (state
->ch
, data
->input
); state->token = new_Token_init
(identify_compound_symbol (state->content), data->start_line,
data->start_pos, state->content);
}
/*
* String literal mode - reading a string
* globals:
* int start_line line on which the current token started
* int start_pos char pos on which the current token started
* params:
* TokeniserState* state current state of the tokeniser
*/
static void string_literal_mode (TokeniserState *state) {
/* a quote terminates the string */
if (state->ch == '"')
state->token = new_Token_init
(TOKEN_STRING, data->start_line, data->start_pos, state->content);
/* a backslash escapes the next character */
else if (state->ch == '\\') {
state->ch = read_character (state);
store_character (state);
state->ch = read_character (state);
}
/* EOF generates an error */
else if (state->ch == EOF)
state->token = new_Token_init
(TOKEN_ILLEGAL, data->start_line, data->start_pos, state->content);
/* all other characters are part of the string */
else {
store_character (state);
state->ch = read_character (state);
}
}
/*
* Top Level Tokeniser Routines
*/
/*
* Get the next token
* params:
* TokenStream* token_stream the token stream being processed
* returns:
* Token* the token built
*/
static Token *next (TokenStream *token_stream) {
/* local variables */
TokeniserState state; /* current state of reading */
Token *return_token; /* token to return */
/* initialise */
this = token_stream;
data = this->data;
state.token = NULL;
state.mode = DEFAULT_MODE;
state.max = 1024;
state.
content = malloc (state.
max); *(state.content) = '\0';
state.ch = read_character (&state);
/* main loop */
while (state.token == NULL) {
switch (state.mode) {
case DEFAULT_MODE:
default_mode (&state);
break;
case COMMENT_MODE:
comment_mode (&state);
break;
case WORD_MODE:
word_mode (&state);
break;
case NUMBER_MODE:
number_mode (&state);
break;
case LESS_THAN_MODE:
less_than_mode (&state);
break;
case GREATER_THAN_MODE:
greater_than_mode (&state);
break;
case STRING_LITERAL_MODE:
string_literal_mode (&state);
break;
default:
state.token = new_Token_init
(TOKEN_EOF, data->start_line, data->start_pos, state.content);
state.ch = EOF; /* temporary hack */
}
}
/* store token and release state memory */
return_token = state.token;
/* return result */
return return_token;
}
/*
* Getter for the current line number
* paramss:
* TokenStream* token_stream the token stream being processed
* returns:
* int the current line number returned
*/
static int get_line (TokenStream *token_stream) {
this = token_stream;
data = this->data;
return data->line;
}
/*
* Destructor for a TokenStream
* params:
* TokenStream* token_stream the doomed token stream
*/
static void destroy (TokenStream *token_stream) {
if (token_stream) {
if (token_stream->data)
free (token_stream
->data
); }
}
/*
* Constructors
*/
/*
* Constructor for TokenStream
* params:
* FILE* input Input file
* returns:
* TokenStream* The new token stream
*/
TokenStream *new_TokenStream (FILE *input) {
/* allocate the memory */
this
= malloc (sizeof (TokenStream
)); this
->data
= data
= malloc (sizeof (Private
));
/* initialise methods */
this->next = next;
this->get_line = get_line;
this->destroy = destroy;
/* initialise data */
data->input = input;
data->line = data->start_line = 1;
data->pos = data->start_pos = 0;
/* return new token stream */
return this;
}