Subversion Repositories Kolibri OS

import tokenize

from tokenize import Token

if '.' in str(1.0):

    from boot import *

def check(t,*vs):

    if vs[0] == None: return True

    if t.type in vs: return True

    if t.type == 'symbol' and t.val in vs: return True

    return False

def tweak(k,v):

    P.stack.append((k,dmap[k]))

    if v: dmap[k] = omap[k]

    else: dmap[k] = {'lbp':0,'nud':itself}

def restore():

    k,v = P.stack.pop()

    dmap[k] = v

def cpy(d):

    r = {}

    for k in d: r[k] = d[k]

    return r

class PData:

    def __init__(self,s,tokens):

        self.s = s

        self.tokens = tokens

        self.pos = 0

        self.token = None

        self.stack = []

    def init(self):

        global omap,dmap

        omap = cpy(base_dmap)

        dmap = cpy(base_dmap)

        self.advance()

    def advance(self,val=None):

        if not check(self.token,val):

            error('expected '+val,self.token)

        if self.pos < len(self.tokens):

            t = self.tokens[self.pos]

            self.pos += 1

        else:

            t = Token((0,0),'eof','eof')

        self.token = do(t)

        return t

def error(ctx,t):

    print t

    tokenize.u_error(ctx,P.s,t.pos)

def nud(t):

    #if 'nud' not in t:

    #    error('no nud',t)

    return t.nud(t)

def led(t,left):

    #if 'led' not in t:

    #    error('no led',t)

    return t.led(t,left)

def get_lbp(t):

    #if 'lbp' not in t:

    #    error('no lbp',t)

    return t.lbp

def get_items(t):

    #if 'items' not in t:

    #    error('no items',t)

    return t.items

def expression(rbp):

    t = P.token

    advance()

    left = nud(t)

    while rbp < get_lbp(P.token):

        t = P.token

        advance()

        left = led(t,left)

    return left

def infix_led(t,left):

    t.items = [left,expression(t.bp)]

    return t

def infix_is(t,left):

    if check(P.token,'not'):

        t.val = 'isnot'

        advance('not')

    t.items = [left,expression(t.bp)]

    return t

def infix_not(t,left):

    advance('in')

    t.val = 'notin'

    t.items = [left,expression(t.bp)]

    return t

def infix_tuple(t,left):

    r = expression(t.bp)

    if left.val == ',':

        left.items.append(r)

        return left

    t.items = [left,r]

    t.type = 'tuple'

    return t

def lst(t):

    if t == None: return []

    if check(t,',','tuple','statements'):

        return get_items(t)

    return [t]

def ilst(typ,t):

    return Token(t.pos,typ,typ,lst(t))

def call_led(t,left):

    r = Token(t.pos,'call','$',[left])

    while not check(P.token,')'):

        tweak(',',0)

        r.items.append(expression(0))

        if P.token.val == ',': advance(',')

        restore()

    advance(")")

    return r

def get_led(t,left):

    r = Token(t.pos,'get','.',[left])

    items =  [left]

    more = False

    while not check(P.token,']'):

        more = False

        if check(P.token,':'):

            items.append(Token(P.token.pos,'symbol','None'))

        else:

            items.append(expression(0))

        if check(P.token,':'):

            advance(':')

            more = True

    if more:

        items.append(Token(P.token.pos,'symbol','None'))

    if len(items) > 2:

        items = [left,Token(t.pos,'slice',':',items[1:])]

    r.items = items

    advance("]")

    return r

def dot_led(t,left):

    r = expression(t.bp)

    r.type = 'string'

    t.items = [left,r]

    return t

def itself(t):

    return t

def paren_nud(t):

    tweak(',',1)

    r = expression(0)

    restore()

    advance(')')

    return r

def list_nud(t):

    t.type = 'list'

    t.val = '[]'

    t.items = []

    next = P.token

    tweak(',',0)

    while not check(P.token,'for',']'):

        r = expression(0)

        t.items.append(r)

        if P.token.val == ',': advance(',')

    if check(P.token,'for'):

        t.type = 'comp'

        advance('for')

        tweak('in',0)

        t.items.append(expression(0))

        advance('in')

        t.items.append(expression(0))

        restore()

    restore()

    advance(']')

    return t

def dict_nud(t):

    t.type='dict'

    t.val = '{}'

    t.items = []

    tweak(',',0)

    while not check(P.token,'}'):

        t.items.append(expression(0))

        if check(P.token,':',','): advance()

    restore()

    advance('}')

    return t

def advance(t=None):

    return P.advance(t)

def block():

    items = []

    tok = P.token

    while check(P.token,'nl'): advance()

    if check(P.token,'indent'):

        advance('indent')

        while not check(P.token,'dedent'):

            items.append(expression(0))

            while check(P.token,';','nl'): advance()

        advance('dedent')

    else:

        items.append(expression(0))

        while check(P.token,';'):

            advance(';')

            items.append(expression(0))

    while check(P.token,'nl'): advance()

    if len(items) > 1:

        return Token(tok.pos,'statements',';',items)

    return items.pop()

def def_nud(t):

    items = t.items = []

    items.append(P.token); advance()

    advance('(')

    r = Token(t.pos,'symbol','():',[])

    items.append(r)

    while not check(P.token,')'):

        tweak(',',0)

        r.items.append(expression(0))

        if check(P.token,','): advance(',')

        restore()

    advance(')')

    advance(':')

    items.append(block())

    return t

def while_nud(t):

    items = t.items = []

    items.append(expression(0))

    advance(':')

    items.append(block())

    return t

def class_nud(t):

    items = t.items = []

    items.append(expression(0))

    advance(':')

    items.append(ilst('methods',block()))

    return t

def from_nud(t):

    items = t.items = []

    items.append(expression(0))

    advance('import')

    items.append(expression(0))

    return t

def for_nud(t):

    items = t.items = []

    tweak('in',0)

    items.append(expression(0))

    advance('in')

    items.append(expression(0))

    restore()

    advance(':')

    items.append(block())

    return t

def if_nud(t):

    items = t.items = []

    a = expression(0)

    advance(':')

    b = block()

    items.append(Token(t.pos,'elif','elif',[a,b]))

    while check(P.token,'elif'):

        tok = P.token

        advance('elif')

        a = expression(0)

        advance(':')

        b = block()

        items.append(Token(tok.pos,'elif','elif',[a,b]))

    if check(P.token,'else'):

        tok = P.token

        advance('else')

        advance(':')

        b = block()

        items.append(Token(tok.pos,'else','else',[b]))

    return t

def try_nud(t):

    items = t.items = []

    advance(':')

    b = block()

    items.append(b)

    while check(P.token,'except'):

        tok = P.token

        advance('except')

        if not check(P.token,':'): a = expression(0)

        else: a = Token(tok.pos,'symbol','None')

        advance(':')

        b = block()

        items.append(Token(tok.pos,'except','except',[a,b]))

    if check(P.token,'else'):

        tok = P.token

        advance('else')

        advance(':')

        b = block()

        items.append(Token(tok.pos,'else','else',[b]))

    return t

def prefix_nud(t):

    #bp = 70

    #if 'bp' in t: bp = t['bp']

    bp = t.bp

    t.items = [expression(bp)]

    return t

def prefix_nud0(t):

    if check(P.token,'nl',';','eof','dedent'): return t

    return prefix_nud(t)

def prefix_nuds(t):

    r = expression(0)

    return ilst(t.type,r)

def prefix_neg(t):

    r = expression(50)

    if r.type == 'number':

        r.val = str(-float(r.val))

        return r

    t.items = [Token(t.pos,'number','0'),r]

    return t

def vargs_nud(t):

    r = prefix_nud(t)

    t.type = 'args'

    t.val = '*'

    return t

def nargs_nud(t):

    r = prefix_nud(t)

    t.type = 'nargs'

    t.val = '**'

    return t

base_dmap = {

    ',':{'lbp':20,'bp':20,'led':infix_tuple},

    '+':{'lbp':50,'bp':50,'led':infix_led},

    '-':{'lbp':50,'nud':prefix_neg,

        'bp':50,'led':infix_led},

    'not':{'lbp':35,'nud':prefix_nud,'bp':35,

        'bp':35,'led':infix_not },

    '%':{'lbp':60,'bp':60,'led':infix_led},

    '*':{'lbp':60,'nud':vargs_nud,

        'bp':60,'led':infix_led,},

    '**': {'lbp':65,'nud':nargs_nud,

        'bp':65,'led':infix_led,},

    '/':{'lbp':60,'bp':60,'led':infix_led},

    '(':{'lbp':70,'nud':paren_nud,

        'bp':80,'led':call_led,},

    '[':{'lbp':70,'nud':list_nud,

        'bp':80,'led':get_led,},

    '{':{'lbp':0,'nud':dict_nud,},

    '.':{'lbp':80,'bp':80,'led':dot_led,'type':'get',},

    'break':{'lbp':0,'nud':itself,'type':'break'},

    'pass':{'lbp':0,'nud':itself,'type':'pass'},

    'continue':{'lbp':0,'nud':itself,'type':'continue'},

    'eof':{'lbp':0,'type':'eof','val':'eof'},

    'def':{'lbp':0,'nud':def_nud,'type':'def',},

    'while':{'lbp':0,'nud':while_nud,'type':'while',},

    'for':{'lbp':0,'nud':for_nud,'type':'for',},

    'try':{'lbp':0,'nud':try_nud,'type':'try',},

    'if':{'lbp':0,'nud':if_nud,'type':'if',},

    'class':{'lbp':0,'nud':class_nud,'type':'class',},

    'raise':{'lbp':0,'nud':prefix_nud0,'type':'raise','bp':20,},

    'return':{'lbp':0,'nud':prefix_nud0,'type':'return','bp':10,},

    'import':{'lbp':0,'nud':prefix_nuds,'type':'import','bp':20,},

    'from':{'lbp':0,'nud':from_nud,'type':'from','bp':20,},

    'del':{'lbp':0,'nud':prefix_nuds,'type':'del','bp':10,},

    'global':{'lbp':0,'nud':prefix_nuds,'type':'globals','bp':20,},

    '=':{

        'lbp':10,'bp':9,'led':infix_led,

        },

}

def i_infix(bp,led,*vs):

    for v in vs: base_dmap[v] = {'lbp':bp,'bp':bp,'led':led}

i_infix(40,infix_led,'<','>','<=','>=','!=','==')

i_infix(40,infix_is,'is','in')

i_infix(10,infix_led,'+=','-=','*=','/=')

i_infix(31,infix_led,'and','&')

i_infix(30,infix_led,'or','|')

i_infix(36,infix_led,'<<','>>')

def i_terms(*vs):

    for v in vs: base_dmap[v] = {'lbp':0,'nud':itself}

i_terms(')','}',']',';',':','nl','elif','else','True','False','None','name','string','number','indent','dedent','except')

base_dmap['nl']['val'] = 'nl'

def gmap(t,v):

    if v not in dmap:

        error('unknown "%s"'%v,t)

    return dmap[v]

def do(t):

    if t.type == 'symbol': r = gmap(t,t.val)

    else: r = gmap(t,t.type)

    merge(t,r)

    return t

def do_module():

    tok = P.token

    items = []

    while not check(P.token,'eof'):

        items.append(block())

    if len(items) > 1:

        return Token(tok.pos,'statements',';',items)

    return items.pop()

def parse(s,tokens,wrap=0):

    global P

    s = tokenize.clean(s)

    P=PData(s,tokens); P.init()

    r = do_module()

    P = None

    return r