| 0,0 → 1,699 |
|---|
| import tokenize |
| from tokenize import Token |
| if '.' in str(1.0): |
| from boot import * |
| |
| EOF,ADD,SUB,MUL,DIV,POW,AND,OR,CMP,GET,SET,NUMBER,STRING,GGET,GSET,MOVE,DEF,PASS,JUMP,CALL,RETURN,IF,DEBUG,EQ,LE,LT,DICT,LIST,NONE,LEN,POS,PARAMS,IGET,FILE,NAME,NE,HAS,RAISE,SETJMP,MOD,LSH,RSH,ITER,DEL,REGS = 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44 |
| |
| class DState: |
| def __init__(self,code,fname): |
| self.code, self.fname = code,fname |
| self.lines = self.code.split('\n') |
| |
| self.stack,self.out,self._scopei,self.tstack,self._tagi,self.data = [],[('tag','EOF')],0,[],0,{} |
| self.error = False |
| def begin(self,gbl=False): |
| if len(self.stack): self.stack.append((self.vars,self.r2n,self.n2r,self._tmpi,self.mreg,self.snum,self._globals,self.lineno,self.globals,self.cregs,self.tmpc)) |
| else: self.stack.append(None) |
| self.vars,self.r2n,self.n2r,self._tmpi,self.mreg,self.snum,self._globals,self.lineno,self.globals,self.cregs,self.tmpc = [],{},{},0,0,str(self._scopei),gbl,-1,[],['regs'],0 |
| self._scopei += 1 |
| insert(self.cregs) |
| def end(self): |
| self.cregs.append(self.mreg) |
| code(EOF) |
| |
| # This next line forces the encoder to |
| # throw an exception if any tmp regs |
| # were leaked within the frame |
| assert(self.tmpc == 0) #REG |
| |
| if len(self.stack) > 1: |
| self.vars,self.r2n,self.n2r,self._tmpi,self.mreg,self.snum,self._globals,self.lineno,self.globals,self.cregs,self.tmpc = self.stack.pop() |
| else: self.stack.pop() |
| |
| |
| def insert(v): D.out.append(v) |
| def write(v): |
| if istype(v,'list'): |
| insert(v) |
| return |
| for n in range(0,len(v),4): |
| insert(('data',v[n:n+4])) |
| def setpos(v): |
| if '-nopos' in ARGV: return |
| line,x = v |
| if line == D.lineno: return |
| text = D.lines[line-1] |
| D.lineno = line |
| val = text + "\0"*(4-len(text)%4) |
| code_16(POS,len(val)/4,line) |
| write(val) |
| def code(i,a=0,b=0,c=0): |
| if not istype(i,'number'): raise |
| if not istype(a,'number'): raise |
| if not istype(b,'number'): raise |
| if not istype(c,'number'): raise |
| write(('code',i,a,b,c)) |
| def code_16(i,a,b): |
| if b < 0: b += 0x8000 |
| code(i,a,(b&0xff00)>>8,(b&0xff)>>0) |
| def get_code16(i,a,b): |
| return ('code',i,a,(b&0xff00)>>8,(b&0xff)>>0) |
| |
| def _do_string(v,r=None): |
| r = get_tmp(r) |
| val = v + "\0"*(4-len(v)%4) |
| code_16(STRING,r,len(v)) |
| write(val) |
| return r |
| def do_string(t,r=None): |
| return _do_string(t.val,r) |
| |
| def _do_number(v,r=None): |
| r = get_tmp(r) |
| code(NUMBER,r,0,0) |
| write(fpack(number(v))) |
| return r |
| def do_number(t,r=None): |
| return _do_number(t.val,r) |
| |
| def get_tag(): |
| k = str(D._tagi) |
| D._tagi += 1 |
| return k |
| def stack_tag(): |
| k = get_tag() |
| D.tstack.append(k) |
| return k |
| def pop_tag(): |
| D.tstack.pop() |
| |
| def tag(*t): |
| t = D.snum+':'+':'.join([str(v) for v in t]) |
| insert(('tag',t)) |
| def jump(*t): |
| t = D.snum+':'+':'.join([str(v) for v in t]) |
| insert(('jump',t)) |
| def setjmp(*t): |
| t = D.snum+':'+':'.join([str(v) for v in t]) |
| insert(('setjmp',t)) |
| def fnc(*t): |
| t = D.snum+':'+':'.join([str(v) for v in t]) |
| r = get_reg(t) |
| insert(('fnc',r,t)) |
| return r |
| |
| def map_tags(): |
| tags = {} |
| out = [] |
| n = 0 |
| for item in D.out: |
| if item[0] == 'tag': |
| tags[item[1]] = n |
| continue |
| if item[0] == 'regs': |
| out.append(get_code16(REGS,item[1],0)) |
| n += 1 |
| continue |
| out.append(item) |
| n += 1 |
| for n in range(0,len(out)): |
| item = out[n] |
| if item[0] == 'jump': |
| out[n] = get_code16(JUMP,0,tags[item[1]]-n) |
| elif item[0] == 'setjmp': |
| out[n] = get_code16(SETJMP,0,tags[item[1]]-n) |
| elif item[0] == 'fnc': |
| out[n] = get_code16(DEF,item[1],tags[item[2]]-n) |
| for n in range(0,len(out)): |
| item = out[n] |
| if item[0] == 'data': |
| out[n] = item[1] |
| elif item[0] == 'code': |
| i,a,b,c = item[1:] |
| out[n] = chr(i)+chr(a)+chr(b)+chr(c) |
| else: |
| raise str(('huh?',item)) |
| if len(out[n]) != 4: |
| raise ('code '+str(n)+' is wrong length '+str(len(out[n]))) |
| D.out = out |
| |
| def get_tmp(r=None): |
| if r != None: return r |
| return get_tmps(1)[0] |
| def get_tmps(t): |
| rs = alloc(t) |
| regs = range(rs,rs+t) |
| for r in regs: |
| set_reg(r,"$"+str(D._tmpi)) |
| D._tmpi += 1 |
| D.tmpc += t #REG |
| return regs |
| def alloc(t): |
| s = ''.join(["01"[r in D.r2n] for r in range(0,min(256,D.mreg+t))]) |
| return s.index('0'*t) |
| def is_tmp(r): |
| if r is None: return False |
| return (D.r2n[r][0] == '$') |
| def un_tmp(r): |
| n = D.r2n[r] |
| free_reg(r) |
| set_reg(r,'*'+n) |
| def free_tmp(r): |
| if is_tmp(r): free_reg(r) |
| return r |
| def free_tmps(r): |
| for k in r: free_tmp(k) |
| def get_reg(n): |
| if n not in D.n2r: |
| set_reg(alloc(1),n) |
| return D.n2r[n] |
| #def get_clean_reg(n): |
| #if n in D.n2r: raise |
| #set_reg(D.mreg,n) |
| #return D.n2r[n] |
| def set_reg(r,n): |
| D.n2r[n] = r; D.r2n[r] = n |
| D.mreg = max(D.mreg,r+1) |
| def free_reg(r): |
| if is_tmp(r): D.tmpc -= 1 |
| n = D.r2n[r]; del D.r2n[r]; del D.n2r[n] |
| |
| def imanage(orig,fnc): |
| items = orig.items |
| orig.val = orig.val[:-1] |
| t = Token(orig.pos,'symbol','=',[items[0],orig]) |
| return fnc(t) |
| |
| def infix(i,tb,tc,r=None): |
| r = get_tmp(r) |
| b,c = do(tb,r),do(tc) |
| code(i,r,b,c) |
| if r != b: free_tmp(b) |
| free_tmp(c) |
| return r |
| def ss_infix(ss,i,tb,tc,r=None): |
| r = get_tmp(r) |
| r2 = get_tmp() |
| ss = _do_number(ss) |
| t = get_tag() |
| r = do(tb,r) |
| code(EQ,r2,r,ss) |
| code(IF,r2) |
| jump(t,'else') |
| jump(t,'end') |
| tag(t,'else') |
| r = do(tc,r) |
| tag(t,'end') |
| free_tmp(r2) #REG |
| free_tmp(ss) #REG |
| return r |
| |
| def _do_none(r=None): |
| r = get_tmp(r) |
| code(NONE,r) |
| return r |
| |
| def do_symbol(t,r=None): |
| sets = ['='] |
| isets = ['+=','-=','*=','/='] |
| cmps = ['<','>','<=','>=','==','!='] |
| metas = { |
| '+':ADD,'*':MUL,'/':DIV,'**':POW, |
| '-':SUB,'and':AND,'or':OR, |
| '%':MOD,'>>':RSH,'<<':LSH, |
| '&':AND,'|':OR, |
| } |
| if t.val == 'None': return _do_none(r) |
| if t.val == 'True': |
| return _do_number('1',r) |
| if t.val == 'False': |
| return _do_number('0',r) |
| items = t.items |
| |
| if t.val in ['and','or']: |
| ss = int(t.val == 'or') |
| return ss_infix(ss,metas[t.val],items[0],items[1],r) |
| if t.val in isets: |
| return imanage(t,do_symbol) |
| if t.val == 'is': |
| return infix(EQ,items[0],items[1],r) |
| if t.val == 'isnot': |
| return infix(CMP,items[0],items[1],r) |
| if t.val == 'not': |
| return infix(EQ,Token(t.pos,'number',0),items[0],r) |
| if t.val == 'in': |
| return infix(HAS,items[1],items[0],r) |
| if t.val == 'notin': |
| r = infix(HAS,items[1],items[0],r) |
| zero = _do_number('0') |
| code(EQ,r,r,free_tmp(zero)) |
| return r |
| if t.val in sets: |
| return do_set_ctx(items[0],items[1]); |
| elif t.val in cmps: |
| b,c = items[0],items[1] |
| v = t.val |
| if v[0] in ('>','>='): |
| b,c,v = c,b,'<'+v[1:] |
| cd = EQ |
| if v == '<': cd = LT |
| if v == '<=': cd = LE |
| if v == '!=': cd = NE |
| return infix(cd,b,c,r) |
| else: |
| return infix(metas[t.val],items[0],items[1],r) |
| |
| def do_set_ctx(k,v): |
| if k.type == 'name': |
| if (D._globals and k.val not in D.vars) or (k.val in D.globals): |
| c = do_string(k) |
| b = do(v) |
| code(GSET,c,b) |
| free_tmp(c) |
| free_tmp(b) |
| return |
| a = do_local(k) |
| b = do(v) |
| code(MOVE,a,b) |
| free_tmp(b) |
| return a |
| elif k.type in ('tuple','list'): |
| if v.type in ('tuple','list'): |
| n,tmps = 0,[] |
| for kk in k.items: |
| vv = v.items[n] |
| tmp = get_tmp(); tmps.append(tmp) |
| r = do(vv) |
| code(MOVE,tmp,r) |
| free_tmp(r) #REG |
| n+=1 |
| n = 0 |
| for kk in k.items: |
| vv = v.items[n] |
| tmp = tmps[n] |
| free_tmp(do_set_ctx(kk,Token(vv.pos,'reg',tmp))) #REG |
| n += 1 |
| return |
| |
| r = do(v); un_tmp(r) |
| n, tmp = 0, Token(v.pos,'reg',r) |
| for tt in k.items: |
| free_tmp(do_set_ctx(tt,Token(tmp.pos,'get',None,[tmp,Token(tmp.pos,'number',str(n))]))) #REG |
| n += 1 |
| free_reg(r) |
| return |
| r = do(k.items[0]) |
| rr = do(v) |
| tmp = do(k.items[1]) |
| code(SET,r,tmp,rr) |
| free_tmp(r) #REG |
| free_tmp(tmp) #REG |
| return rr |
| |
| def manage_seq(i,a,items,sav=0): |
| l = max(sav,len(items)) |
| n,tmps = 0,get_tmps(l) |
| for tt in items: |
| r = tmps[n] |
| b = do(tt,r) |
| if r != b: |
| code(MOVE,r,b) |
| free_tmp(b) |
| n +=1 |
| if not len(tmps): |
| code(i,a,0,0) |
| return 0 |
| code(i,a,tmps[0],len(items)) |
| free_tmps(tmps[sav:]) |
| return tmps[0] |
| |
| def p_filter(items): |
| a,b,c,d = [],[],None,None |
| for t in items: |
| if t.type == 'symbol' and t.val == '=': b.append(t) |
| elif t.type == 'args': c = t |
| elif t.type == 'nargs': d = t |
| else: a.append(t) |
| return a,b,c,d |
| |
| def do_import(t): |
| for mod in t.items: |
| mod.type = 'string' |
| v = do_call(Token(t.pos,'call',None,[ |
| Token(t.pos,'name','import'), |
| mod])) |
| mod.type = 'name' |
| do_set_ctx(mod,Token(t.pos,'reg',v)) |
| def do_from(t): |
| mod = t.items[0] |
| mod.type = 'string' |
| v = do(Token(t.pos,'call',None,[ |
| Token(t.pos,'name','import'), |
| mod])) |
| item = t.items[1] |
| if item.val == '*': |
| free_tmp(do(Token(t.pos,'call',None,[ |
| Token(t.pos,'name','merge'), |
| Token(t.pos,'name','__dict__'), |
| Token(t.pos,'reg',v)]))) #REG |
| else: |
| item.type = 'string' |
| free_tmp(do_set_ctx( |
| Token(t.pos,'get',None,[ Token(t.pos,'name','__dict__'),item]), |
| Token(t.pos,'get',None,[ Token(t.pos,'reg',v),item]) |
| )) #REG |
| |
| |
| def do_globals(t): |
| for t in t.items: |
| if t.val not in D.globals: |
| D.globals.append(t.val) |
| def do_del(tt): |
| for t in tt.items: |
| r = do(t.items[0]) |
| r2 = do(t.items[1]) |
| code(DEL,r,r2) |
| free_tmp(r); free_tmp(r2) #REG |
| |
| def do_call(t,r=None): |
| r = get_tmp(r) |
| items = t.items |
| fnc = do(items[0]) |
| a,b,c,d = p_filter(t.items[1:]) |
| e = None |
| if len(b) != 0 or d != None: |
| e = do(Token(t.pos,'dict',None,[])); un_tmp(e); |
| for p in b: |
| p.items[0].type = 'string' |
| t1,t2 = do(p.items[0]),do(p.items[1]) |
| code(SET,e,t1,t2) |
| free_tmp(t1); free_tmp(t2) #REG |
| if d: free_tmp(do(Token(t.pos,'call',None,[Token(t.pos,'name','merge'),Token(t.pos,'reg',e),d.items[0]]))) #REG |
| manage_seq(PARAMS,r,a) |
| if c != None: |
| t1,t2 = _do_string('*'),do(c.items[0]) |
| code(SET,r,t1,t2) |
| free_tmp(t1); free_tmp(t2) #REG |
| if e != None: |
| t1 = _do_none() |
| code(SET,r,t1,e) |
| free_tmp(t1) #REG |
| code(CALL,r,fnc,r) |
| free_tmp(fnc) #REG |
| return r |
| |
| def do_name(t,r=None): |
| if t.val in D.vars: |
| return do_local(t,r) |
| r = get_tmp(r) |
| c = do_string(t) |
| code(GGET,r,c) |
| free_tmp(c) |
| return r |
| |
| def do_local(t,r=None): |
| if t.val not in D.vars: |
| D.vars.append(t.val) |
| return get_reg(t.val) |
| |
| def do_def(tok,kls=None): |
| items = tok.items |
| |
| t = get_tag() |
| rf = fnc(t,'end') |
| |
| D.begin() |
| setpos(tok.pos) |
| r = do_local(Token(tok.pos,'name','__params')) |
| do_info(items[0].val) |
| a,b,c,d = p_filter(items[1].items) |
| for p in a: |
| v = do_local(p) |
| tmp = _do_none() |
| code(GET,v,r,tmp) |
| free_tmp(tmp) #REG |
| for p in b: |
| v = do_local(p.items[0]) |
| do(p.items[1],v) |
| tmp = _do_none() |
| code(IGET,v,r,tmp) |
| free_tmp(tmp) #REG |
| if c != None: |
| v = do_local(c.items[0]) |
| tmp = _do_string('*') |
| code(GET,v,r,tmp) |
| free_tmp(tmp) #REG |
| if d != None: |
| e = do_local(d.items[0]) |
| code(DICT,e,0,0) |
| tmp = _do_none() |
| code(IGET,e,r,tmp) |
| free_tmp(tmp) #REG |
| free_tmp(do(items[2])) #REG |
| D.end() |
| |
| tag(t,'end') |
| |
| if kls == None: |
| if D._globals: do_globals(Token(tok.pos,0,0,[items[0]])) |
| r = do_set_ctx(items[0],Token(tok.pos,'reg',rf)) |
| else: |
| rn = do_string(items[0]) |
| code(SET,kls,rn,rf) |
| free_tmp(rn) |
| |
| free_tmp(rf) |
| |
| def do_class(t): |
| tok = t |
| items = t.items |
| parent = None |
| if items[0].type == 'name': |
| name = items[0].val |
| else: |
| name = items[0].items[0].val |
| parent = items[0].items[1].val |
| |
| kls = do(Token(t.pos,'dict',0,[])) |
| ts = _do_string(name) |
| code(GSET,ts,kls) |
| free_tmp(ts) #REG |
| |
| init,_new = False,[] |
| if parent: |
| _new.append(Token(t.pos,'call',None,[ |
| Token(t.pos,'get',None,[ |
| Token(t.pos,'name',parent), |
| Token(t.pos,'string','__new__'), |
| ]), |
| Token(t.pos,'name','self'), |
| ])) |
| |
| for fc in items[1].items: |
| if fc.type != 'def': continue |
| fn = fc.items[0].val |
| if fn == '__init__': init = True |
| do_def(fc,kls) |
| _new.append(Token(fc.pos,'symbol','=',[ |
| Token(fc.pos,'get',None,[ |
| Token(fc.pos,'name','self'), |
| Token(fc.pos,'string',fn)]), |
| Token(fc.pos,'call',None,[ |
| Token(fc.pos,'name','bind'), |
| Token(fc.pos,'get',None,[ |
| Token(fc.pos,'name',name), |
| Token(fc.pos,'string',fn)]), |
| Token(fc.pos,'name','self')]) |
| ])) |
| |
| do_def(Token(t.pos,'def',None,[ |
| Token(t.pos,'name','__new__'), |
| Token(t.pos,'list',None,[Token(t.pos,'name','self')]), |
| Token(t.pos,'statements',None,_new)]),kls) |
| |
| t = get_tag() |
| rf = fnc(t,'end') |
| D.begin() |
| params = do_local(Token(tok.pos,'name','__params')) |
| |
| slf = do_local(Token(tok.pos,'name','self')) |
| code(DICT,slf,0,0) |
| |
| free_tmp(do(Token(tok.pos,'call',None,[ |
| Token(tok.pos,'get',None,[ |
| Token(tok.pos,'name',name), |
| Token(tok.pos,'string','__new__')]), |
| Token(tok.pos,'name','self')]))) #REG |
| |
| if init: |
| tmp = get_tmp() |
| t3 = _do_string('__init__') |
| code(GET,tmp,slf,t3) |
| t4 = get_tmp() |
| code(CALL,t4,tmp,params) |
| free_tmp(tmp) #REG |
| free_tmp(t3) #REG |
| free_tmp(t4) #REG |
| code(RETURN,slf) |
| |
| D.end() |
| tag(t,'end') |
| ts = _do_string('__call__') |
| code(SET,kls,ts,rf) |
| free_tmp(kls) #REG |
| free_tmp(ts) #REG |
| |
| |
| |
| |
| def do_while(t): |
| items = t.items |
| t = stack_tag() |
| tag(t,'begin') |
| tag(t,'continue') |
| r = do(items[0]) |
| code(IF,r) |
| free_tmp(r) #REG |
| jump(t,'end') |
| free_tmp(do(items[1])) #REG |
| jump(t,'begin') |
| tag(t,'break') |
| tag(t,'end') |
| pop_tag() |
| |
| def do_for(tok): |
| items = tok.items |
| |
| reg = do_local(items[0]) |
| itr = do(items[1]) |
| i = _do_number('0') |
| |
| t = stack_tag(); tag(t,'loop'); tag(t,'continue') |
| code(ITER,reg,itr,i); jump(t,'end') |
| free_tmp(do(items[2])) #REG |
| jump(t,'loop') |
| tag(t,'break'); tag(t,'end'); pop_tag() |
| |
| free_tmp(itr) #REG |
| free_tmp(i) |
| |
| def do_comp(t,r=None): |
| name = 'comp:'+get_tag() |
| r = do_local(Token(t.pos,'name',name)) |
| code(LIST,r,0,0) |
| key = Token(t.pos,'get',None,[ |
| Token(t.pos,'reg',r), |
| Token(t.pos,'symbol','None')]) |
| ap = Token(t.pos,'symbol','=',[key,t.items[0]]) |
| do(Token(t.pos,'for',None,[t.items[1],t.items[2],ap])) |
| return r |
| |
| def do_if(t): |
| items = t.items |
| t = get_tag() |
| n = 0 |
| for tt in items: |
| tag(t,n) |
| if tt.type == 'elif': |
| a = do(tt.items[0]); code(IF,a); free_tmp(a); |
| jump(t,n+1) |
| free_tmp(do(tt.items[1])) #REG |
| elif tt.type == 'else': |
| free_tmp(do(tt.items[0])) #REG |
| else: |
| raise |
| jump(t,'end') |
| n += 1 |
| tag(t,n) |
| tag(t,'end') |
| |
| def do_try(t): |
| items = t.items |
| t = get_tag() |
| setjmp(t,'except') |
| free_tmp(do(items[0])) #REG |
| jump(t,'end') |
| tag(t,'except') |
| free_tmp(do(items[1].items[1])) #REG |
| tag(t,'end') |
| |
| def do_return(t): |
| if t.items: r = do(t.items[0]) |
| else: r = _do_none() |
| code(RETURN,r) |
| free_tmp(r) |
| return |
| def do_raise(t): |
| if t.items: r = do(t.items[0]) |
| else: r = _do_none() |
| code(RAISE,r) |
| free_tmp(r) |
| return |
| |
| def do_statements(t): |
| for tt in t.items: free_tmp(do(tt)) |
| |
| def do_list(t,r=None): |
| r = get_tmp(r) |
| manage_seq(LIST,r,t.items) |
| return r |
| |
| def do_dict(t,r=None): |
| r = get_tmp(r) |
| manage_seq(DICT,r,t.items) |
| return r |
| |
| def do_get(t,r=None): |
| items = t.items |
| return infix(GET,items[0],items[1],r) |
| |
| def do_break(t): jump(D.tstack[-1],'break') |
| def do_continue(t): jump(D.tstack[-1],'continue') |
| def do_pass(t): code(PASS) |
| |
| def do_info(name='?'): |
| if '-nopos' in ARGV: return |
| code(FILE,free_tmp(_do_string(D.fname))) |
| code(NAME,free_tmp(_do_string(name))) |
| def do_module(t): |
| do_info() |
| free_tmp(do(t.items[0])) #REG |
| def do_reg(t,r=None): return t.val |
| |
| fmap = { |
| 'module':do_module,'statements':do_statements,'def':do_def, |
| 'return':do_return,'while':do_while,'if':do_if, |
| 'break':do_break,'pass':do_pass,'continue':do_continue,'for':do_for, |
| 'class':do_class,'raise':do_raise,'try':do_try,'import':do_import, |
| 'globals':do_globals,'del':do_del,'from':do_from, |
| } |
| rmap = { |
| 'list':do_list, 'tuple':do_list, 'dict':do_dict, 'slice':do_list, |
| 'comp':do_comp, 'name':do_name,'symbol':do_symbol,'number':do_number, |
| 'string':do_string,'get':do_get, 'call':do_call, 'reg':do_reg, |
| } |
| |
| def do(t,r=None): |
| if t.pos: setpos(t.pos) |
| try: |
| if t.type in rmap: |
| return rmap[t.type](t,r) |
| return fmap[t.type](t) |
| except: |
| if D.error: raise |
| D.error = True |
| tokenize.u_error('encode',D.code,t.pos) |
| |
| def encode(fname,s,t): |
| t = Token((1,1),'module','module',[t]) |
| global D |
| s = tokenize.clean(s) |
| D = DState(s,fname) |
| D.begin(True) |
| do(t) |
| D.end() |
| map_tags() |
| out = D.out; D = None |
| return ''.join(out) |
| |