add != operator
[soc.git] / src / soc / decoder / pseudo / parser.py
1 # Based on GardenSnake - a parser generator demonstration program
2 # GardenSnake was released into the Public Domain by Andrew Dalke.
3
4 # Portions of this work are derived from Python's Grammar definition
5 # and may be covered under the Python copyright and license
6 #
7 # Andrew Dalke / Dalke Scientific Software, LLC
8 # 30 August 2006 / Cape Town, South Africa
9
10 # Modifications for inclusion in PLY distribution
11 from pprint import pprint
12 from ply import lex, yacc
13 import astor
14
15 from soc.decoder.power_decoder import create_pdecode
16 from soc.decoder.pseudo.lexer import IndentLexer
17 from soc.decoder.orderedset import OrderedSet
18
19 # I use the Python AST
20 #from compiler import ast
21 import ast
22
23 # Helper function
24
25
26 def Assign(left, right, iea_mode):
27 names = []
28 print("Assign", left, right)
29 if isinstance(left, ast.Name):
30 # Single assignment on left
31 # XXX when doing IntClass, which will have an "eq" function,
32 # this is how to access it
33 # eq = ast.Attribute(left, "eq") # get eq fn
34 # return ast.Call(eq, [right], []) # now call left.eq(right)
35 return ast.Assign([ast.Name(left.id, ast.Store())], right)
36 elif isinstance(left, ast.Tuple):
37 # List of things - make sure they are Name nodes
38 names = []
39 for child in left.getChildren():
40 if not isinstance(child, ast.Name):
41 raise SyntaxError("that assignment not supported")
42 names.append(child.name)
43 ass_list = [ast.AssName(name, 'OP_ASSIGN') for name in names]
44 return ast.Assign([ast.AssTuple(ass_list)], right)
45 elif isinstance(left, ast.Subscript):
46 return ast.Assign([left], right)
47 # XXX HMMM probably not needed...
48 ls = left.slice
49 if isinstance(ls, ast.Slice):
50 lower, upper, step = ls.lower, ls.upper, ls.step
51 print("slice assign", lower, upper, step)
52 if step is None:
53 ls = (lower, upper, None)
54 else:
55 ls = (lower, upper, step)
56 ls = ast.Tuple(ls)
57 return ast.Call(ast.Name("selectassign"),
58 [left.value, ls, right], [])
59 else:
60 print("Assign fail")
61 raise SyntaxError("Can't do that yet")
62
63
64 # I implemented INDENT / DEDENT generation as a post-processing filter
65
66 # The original lex token stream contains WS and NEWLINE characters.
67 # WS will only occur before any other tokens on a line.
68
69 # I have three filters. One tags tokens by adding two attributes.
70 # "must_indent" is True if the token must be indented from the
71 # previous code. The other is "at_line_start" which is True for WS
72 # and the first non-WS/non-NEWLINE on a line. It flags the check so
73 # see if the new line has changed indication level.
74
75
76 # No using Python's approach because Ply supports precedence
77
78 # comparison: expr (comp_op expr)*
79 # arith_expr: term (('+'|'-') term)*
80 # term: factor (('*'|'/'|'%'|'//') factor)*
81 # factor: ('+'|'-'|'~') factor | power
82 # comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
83
84 def make_le_compare(arg):
85 (left, right) = arg
86 return ast.Compare(left, [ast.LtE()], [right])
87
88
89 def make_ge_compare(arg):
90 (left, right) = arg
91 return ast.Compare(left, [ast.GtE()], [right])
92
93
94 def make_lt_compare(arg):
95 (left, right) = arg
96 return ast.Compare(left, [ast.Lt()], [right])
97
98
99 def make_gt_compare(arg):
100 (left, right) = arg
101 return ast.Compare(left, [ast.Gt()], [right])
102
103
104 def make_eq_compare(arg):
105 (left, right) = arg
106 return ast.Compare(left, [ast.Eq()], [right])
107
108
109 def make_ne_compare(arg):
110 (left, right) = arg
111 return ast.Compare(left, [ast.NotEq()], [right])
112
113
114 binary_ops = {
115 "^": ast.BitXor(),
116 "&": ast.BitAnd(),
117 "|": ast.BitOr(),
118 "+": ast.Add(),
119 "-": ast.Sub(),
120 "*": ast.Mult(),
121 "/": ast.Div(),
122 "%": ast.Mod(),
123 "<=": make_le_compare,
124 ">=": make_ge_compare,
125 "<": make_lt_compare,
126 ">": make_gt_compare,
127 "=": make_eq_compare,
128 "!=": make_ne_compare,
129 }
130 unary_ops = {
131 "+": ast.UAdd(),
132 "-": ast.USub(),
133 "¬": ast.Invert(),
134 }
135
136
137 def check_concat(node): # checks if the comparison is already a concat
138 print("check concat", node)
139 if not isinstance(node, ast.Call):
140 return [node]
141 print("func", node.func.id)
142 if node.func.id != 'concat':
143 return [node]
144 if node.keywords: # a repeated list-constant, don't optimise
145 return [node]
146 return node.args
147
148
149 # identify SelectableInt pattern
150 def identify_sint_mul_pattern(p):
151 if not isinstance(p[3], ast.Constant):
152 return False
153 if not isinstance(p[1], ast.List):
154 return False
155 l = p[1].elts
156 if len(l) != 1:
157 return False
158 elt = l[0]
159 return isinstance(elt, ast.Constant)
160
161 def apply_trailer(atom, trailer):
162 if trailer[0] == "TLIST":
163 # assume depth of one
164 atom = apply_trailer(atom, trailer[1])
165 trailer = trailer[2]
166 if trailer[0] == "CALL":
167 #p[0] = ast.Expr(ast.Call(p[1], p[2][1], []))
168 return ast.Call(atom, trailer[1], [])
169 # if p[1].id == 'print':
170 # p[0] = ast.Printnl(ast.Tuple(p[2][1]), None, None)
171 # else:
172 # p[0] = ast.CallFunc(p[1], p[2][1], None, None)
173 else:
174 print("subscript atom", trailer[1])
175 #raise AssertionError("not implemented %s" % p[2][0])
176 subs = trailer[1]
177 if len(subs) == 1:
178 idx = subs[0]
179 else:
180 idx = ast.Slice(subs[0], subs[1], None)
181 return ast.Subscript(atom, idx, ast.Load())
182
183 ########## Parser (tokens -> AST) ######
184
185 # also part of Ply
186 #import yacc
187
188 # https://www.mathcs.emory.edu/~valerie/courses/fall10/155/resources/op_precedence.html
189 # python operator precedence
190 # Highest precedence at top, lowest at bottom.
191 # Operators in the same box evaluate left to right.
192 #
193 # Operator Description
194 # () Parentheses (grouping)
195 # f(args...) Function call
196 # x[index:index] Slicing
197 # x[index] Subscription
198 # x.attribute Attribute reference
199 # ** Exponentiation
200 # ~x Bitwise not
201 # +x, -x Positive, negative
202 # *, /, % mul, div, remainder
203 # +, - Addition, subtraction
204 # <<, >> Bitwise shifts
205 # & Bitwise AND
206 # ^ Bitwise XOR
207 # | Bitwise OR
208 # in, not in, is, is not, <, <=, >, >=, <>, !=, == comp, membership, ident
209 # not x Boolean NOT
210 # and Boolean AND
211 # or Boolean OR
212 # lambda Lambda expression
213
214 class PowerParser:
215
216 precedence = (
217 ("left", "EQ", "NE", "GT", "LT", "LE", "GE", "LTU", "GTU"),
218 ("left", "BITOR"),
219 ("left", "BITXOR"),
220 ("left", "BITAND"),
221 ("left", "PLUS", "MINUS"),
222 ("left", "MULT", "DIV", "MOD"),
223 ("left", "INVERT"),
224 )
225
226 def __init__(self):
227 self.gprs = {}
228 for rname in ['RA', 'RB', 'RC', 'RT', 'RS']:
229 self.gprs[rname] = None
230 self.read_regs = OrderedSet()
231 self.uninit_regs = OrderedSet()
232 self.write_regs = OrderedSet()
233
234 # The grammar comments come from Python's Grammar/Grammar file
235
236 # NB: compound_stmt in single_input is followed by extra NEWLINE!
237 # file_input: (NEWLINE | stmt)* ENDMARKER
238
239 def p_file_input_end(self, p):
240 """file_input_end : file_input ENDMARKER"""
241 print("end", p[1])
242 p[0] = p[1]
243
244 def p_file_input(self, p):
245 """file_input : file_input NEWLINE
246 | file_input stmt
247 | NEWLINE
248 | stmt"""
249 if isinstance(p[len(p)-1], str):
250 if len(p) == 3:
251 p[0] = p[1]
252 else:
253 p[0] = [] # p == 2 --> only a blank line
254 else:
255 if len(p) == 3:
256 p[0] = p[1] + p[2]
257 else:
258 p[0] = p[1]
259
260 # funcdef: [decorators] 'def' NAME parameters ':' suite
261 # ignoring decorators
262
263 def p_funcdef(self, p):
264 "funcdef : DEF NAME parameters COLON suite"
265 p[0] = ast.FunctionDef(p[2], p[3], p[5], ())
266
267 # parameters: '(' [varargslist] ')'
268 def p_parameters(self, p):
269 """parameters : LPAR RPAR
270 | LPAR varargslist RPAR"""
271 if len(p) == 3:
272 args = []
273 else:
274 args = p[2]
275 p[0] = ast.arguments(args=args, vararg=None, kwarg=None, defaults=[])
276
277 # varargslist: (fpdef ['=' test] ',')* ('*' NAME [',' '**' NAME] |
278 # '**' NAME) |
279 # highly simplified
280
281 def p_varargslist(self, p):
282 """varargslist : varargslist COMMA NAME
283 | NAME"""
284 if len(p) == 4:
285 p[0] = p[1] + p[3]
286 else:
287 p[0] = [p[1]]
288
289 # stmt: simple_stmt | compound_stmt
290 def p_stmt_simple(self, p):
291 """stmt : simple_stmt"""
292 # simple_stmt is a list
293 p[0] = p[1]
294
295 def p_stmt_compound(self, p):
296 """stmt : compound_stmt"""
297 p[0] = [p[1]]
298
299 # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
300 def p_simple_stmt(self, p):
301 """simple_stmt : small_stmts NEWLINE
302 | small_stmts SEMICOLON NEWLINE"""
303 p[0] = p[1]
304
305 def p_small_stmts(self, p):
306 """small_stmts : small_stmts SEMICOLON small_stmt
307 | small_stmt"""
308 if len(p) == 4:
309 p[0] = p[1] + [p[3]]
310 else:
311 p[0] = [p[1]]
312
313 # small_stmt: expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |
314 # import_stmt | global_stmt | exec_stmt | assert_stmt
315 def p_small_stmt(self, p):
316 """small_stmt : flow_stmt
317 | break_stmt
318 | expr_stmt"""
319 if isinstance(p[1], ast.Call):
320 p[0] = ast.Expr(p[1])
321 else:
322 p[0] = p[1]
323
324 # expr_stmt: testlist (augassign (yield_expr|testlist) |
325 # ('=' (yield_expr|testlist))*)
326 # augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
327 # '<<=' | '>>=' | '**=' | '//=')
328 def p_expr_stmt(self, p):
329 """expr_stmt : testlist ASSIGNEA testlist
330 | testlist ASSIGN testlist
331 | testlist """
332 print("expr_stmt", p)
333 if len(p) == 2:
334 # a list of expressions
335 #p[0] = ast.Discard(p[1])
336 p[0] = p[1]
337 else:
338 iea_mode = p[2] == '<-iea'
339 name = None
340 if isinstance(p[1], ast.Name):
341 name = p[1].id
342 elif isinstance(p[1], ast.Subscript):
343 name = p[1].value.id
344 if name in self.gprs:
345 # add to list of uninitialised
346 self.uninit_regs.add(name)
347 elif isinstance(p[1], ast.Call) and p[1].func.id == 'GPR':
348 print(astor.dump_tree(p[1]))
349 # replace GPR(x) with GPR[x]
350 idx = p[1].args[0]
351 p[1] = ast.Subscript(p[1].func, idx)
352 elif isinstance(p[1], ast.Call) and p[1].func.id == 'MEM':
353 print ("mem assign")
354 print(astor.dump_tree(p[1]))
355 p[1].func.id = "memassign" # change function name to set
356 p[1].args.append(p[3])
357 p[0] = p[1]
358 print ("mem rewrite")
359 print(astor.dump_tree(p[0]))
360 return
361 else:
362 print ("help, help")
363 print(astor.dump_tree(p[1]))
364 print("expr assign", name, p[1])
365 if name and name in self.gprs:
366 self.write_regs.add(name) # add to list of regs to write
367 p[0] = Assign(p[1], p[3], iea_mode)
368
369 def p_flow_stmt(self, p):
370 "flow_stmt : return_stmt"
371 p[0] = p[1]
372
373 # return_stmt: 'return' [testlist]
374 def p_return_stmt(self, p):
375 "return_stmt : RETURN testlist"
376 p[0] = ast.Return(p[2])
377
378 def p_compound_stmt(self, p):
379 """compound_stmt : if_stmt
380 | while_stmt
381 | for_stmt
382 | funcdef
383 """
384 p[0] = p[1]
385
386 def p_break_stmt(self, p):
387 """break_stmt : BREAK
388 """
389 p[0] = ast.Break()
390
391 def p_for_stmt(self, p):
392 """for_stmt : FOR atom EQ test TO test COLON suite
393 | DO atom EQ test TO test COLON suite
394 """
395 # auto-add-one (sigh) due to python range
396 start = p[4]
397 end = ast.BinOp(p[6], ast.Add(), ast.Constant(1))
398 it = ast.Call(ast.Name("range"), [start, end], [])
399 p[0] = ast.For(p[2], it, p[8], [])
400
401 def p_while_stmt(self, p):
402 """while_stmt : DO WHILE test COLON suite ELSE COLON suite
403 | DO WHILE test COLON suite
404 """
405 if len(p) == 6:
406 p[0] = ast.While(p[3], p[5], [])
407 else:
408 p[0] = ast.While(p[3], p[5], p[8])
409
410 def p_if_stmt(self, p):
411 """if_stmt : IF test COLON suite ELSE COLON if_stmt
412 | IF test COLON suite ELSE COLON suite
413 | IF test COLON suite
414 """
415 if len(p) == 8 and isinstance(p[7], ast.If):
416 p[0] = ast.If(p[2], p[4], [p[7]])
417 elif len(p) == 5:
418 p[0] = ast.If(p[2], p[4], [])
419 else:
420 p[0] = ast.If(p[2], p[4], p[7])
421
422 def p_suite(self, p):
423 """suite : simple_stmt
424 | NEWLINE INDENT stmts DEDENT"""
425 if len(p) == 2:
426 p[0] = p[1]
427 else:
428 p[0] = p[3]
429
430 def p_stmts(self, p):
431 """stmts : stmts stmt
432 | stmt"""
433 if len(p) == 3:
434 p[0] = p[1] + p[2]
435 else:
436 p[0] = p[1]
437
438 def p_comparison(self, p):
439 """comparison : comparison PLUS comparison
440 | comparison MINUS comparison
441 | comparison MULT comparison
442 | comparison DIV comparison
443 | comparison MOD comparison
444 | comparison EQ comparison
445 | comparison NE comparison
446 | comparison LE comparison
447 | comparison GE comparison
448 | comparison LTU comparison
449 | comparison GTU comparison
450 | comparison LT comparison
451 | comparison GT comparison
452 | comparison BITOR comparison
453 | comparison BITXOR comparison
454 | comparison BITAND comparison
455 | PLUS comparison
456 | comparison MINUS
457 | INVERT comparison
458 | comparison APPEND comparison
459 | power"""
460 if len(p) == 4:
461 print(list(p))
462 if p[2] == '<u':
463 p[0] = ast.Call(ast.Name("ltu"), (p[1], p[3]), [])
464 elif p[2] == '>u':
465 p[0] = ast.Call(ast.Name("gtu"), (p[1], p[3]), [])
466 elif p[2] == '||':
467 l = check_concat(p[1]) + check_concat(p[3])
468 p[0] = ast.Call(ast.Name("concat"), l, [])
469 elif p[2] in ['<', '>', '=', '<=', '>=', '!=']:
470 p[0] = binary_ops[p[2]]((p[1], p[3]))
471 elif identify_sint_mul_pattern(p):
472 keywords=[ast.keyword(arg='repeat', value=p[3])]
473 l = p[1].elts
474 p[0] = ast.Call(ast.Name("concat"), l, keywords)
475 else:
476 p[0] = ast.BinOp(p[1], binary_ops[p[2]], p[3])
477 elif len(p) == 3:
478 if isinstance(p[2], str) and p[2] == '-':
479 p[0] = ast.UnaryOp(unary_ops[p[2]], p[1])
480 else:
481 p[0] = ast.UnaryOp(unary_ops[p[1]], p[2])
482 else:
483 p[0] = p[1]
484
485 # power: atom trailer* ['**' factor]
486 # trailers enables function calls (and subscripts).
487 # so this is 'trailerlist'
488 def p_power(self, p):
489 """power : atom
490 | atom trailerlist"""
491 if len(p) == 2:
492 p[0] = p[1]
493 else:
494 print("power dump atom")
495 print(astor.dump_tree(p[1]))
496 print("power dump trailerlist")
497 print(astor.dump_tree(p[2]))
498 p[0] = apply_trailer(p[1], p[2])
499
500 def p_atom_name(self, p):
501 """atom : NAME"""
502 p[0] = ast.Name(id=p[1], ctx=ast.Load())
503
504 def p_atom_number(self, p):
505 """atom : BINARY
506 | NUMBER
507 | STRING"""
508 p[0] = ast.Constant(p[1])
509
510 # '[' [listmaker] ']' |
511
512 def p_atom_listmaker(self, p):
513 """atom : LBRACK listmaker RBRACK"""
514 p[0] = p[2]
515
516 def p_listmaker(self, p):
517 """listmaker : test COMMA listmaker
518 | test
519 """
520 if len(p) == 2:
521 p[0] = ast.List([p[1]])
522 else:
523 p[0] = ast.List([p[1]] + p[3].nodes)
524
525 def p_atom_tuple(self, p):
526 """atom : LPAR testlist RPAR"""
527 print("tuple", p[2])
528 print("astor dump")
529 print(astor.dump_tree(p[2]))
530
531 if isinstance(p[2], ast.Name):
532 print("tuple name", p[2].id)
533 if p[2].id in self.gprs:
534 self.read_regs.add(p[2].id) # add to list of regs to read
535 #p[0] = ast.Subscript(ast.Name("GPR"), ast.Str(p[2].id))
536 # return
537 p[0] = p[2]
538 elif isinstance(p[2], ast.BinOp):
539 if isinstance(p[2].left, ast.Name) and \
540 isinstance(p[2].right, ast.Constant) and \
541 p[2].right.value == 0 and \
542 p[2].left.id in self.gprs:
543 rid = p[2].left.id
544 self.read_regs.add(rid) # add to list of regs to read
545 # create special call to GPR.getz
546 gprz = ast.Name("GPR")
547 gprz = ast.Attribute(gprz, "getz") # get testzero function
548 # *sigh* see class GPR. we need index itself not reg value
549 ridx = ast.Name("_%s" % rid)
550 p[0] = ast.Call(gprz, [ridx], [])
551 print("tree", astor.dump_tree(p[0]))
552 else:
553 p[0] = p[2]
554 else:
555 p[0] = p[2]
556
557 def p_trailerlist(self, p):
558 """trailerlist : trailer trailerlist
559 | trailer
560 """
561 if len(p) == 2:
562 p[0] = p[1]
563 else:
564 p[0] = ("TLIST", p[1], p[2])
565
566 # trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
567 def p_trailer(self, p):
568 """trailer : trailer_arglist
569 | trailer_subscript
570 """
571 p[0] = p[1]
572
573 def p_trailer_arglist(self, p):
574 "trailer_arglist : LPAR arglist RPAR"
575 p[0] = ("CALL", p[2])
576
577 def p_trailer_subscript(self, p):
578 "trailer_subscript : LBRACK subscript RBRACK"
579 p[0] = ("SUBS", p[2])
580
581 # subscript: '.' '.' '.' | test | [test] ':' [test]
582
583 def p_subscript(self, p):
584 """subscript : test COLON test
585 | test
586 """
587 if len(p) == 4:
588 # add one to end
589 if isinstance(p[3], ast.Constant):
590 end = ast.Constant(p[3].value+1)
591 else:
592 end = ast.BinOp(p[3], ast.Add(), ast.Constant(1))
593 p[0] = [p[1], end]
594 else:
595 p[0] = [p[1]]
596
597 # testlist: test (',' test)* [',']
598 # Contains shift/reduce error
599
600 def p_testlist(self, p):
601 """testlist : testlist_multi COMMA
602 | testlist_multi """
603 if len(p) == 2:
604 p[0] = p[1]
605 else:
606 # May need to promote singleton to tuple
607 if isinstance(p[1], list):
608 p[0] = p[1]
609 else:
610 p[0] = [p[1]]
611 # Convert into a tuple?
612 if isinstance(p[0], list):
613 p[0] = ast.Tuple(p[0])
614
615 def p_testlist_multi(self, p):
616 """testlist_multi : testlist_multi COMMA test
617 | test"""
618 if len(p) == 2:
619 # singleton
620 p[0] = p[1]
621 else:
622 if isinstance(p[1], list):
623 p[0] = p[1] + [p[3]]
624 else:
625 # singleton -> tuple
626 p[0] = [p[1], p[3]]
627
628 # test: or_test ['if' or_test 'else' test] | lambdef
629 # as I don't support 'and', 'or', and 'not' this works down to 'comparison'
630
631 def p_test(self, p):
632 "test : comparison"
633 p[0] = p[1]
634
635 # arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
636 # | '**' test)
637 # XXX INCOMPLETE: this doesn't allow the trailing comma
638
639 def p_arglist(self, p):
640 """arglist : arglist COMMA argument
641 | argument"""
642 if len(p) == 4:
643 p[0] = p[1] + [p[3]]
644 else:
645 p[0] = [p[1]]
646
647 # argument: test [gen_for] | test '=' test # Really [keyword '='] test
648 def p_argument(self, p):
649 "argument : test"
650 p[0] = p[1]
651
652 def p_error(self, p):
653 # print "Error!", repr(p)
654 raise SyntaxError(p)
655
656
657 class GardenSnakeParser(PowerParser):
658 def __init__(self, lexer=None, debug=False):
659 PowerParser.__init__(self)
660 self.debug = debug
661 if lexer is None:
662 lexer = IndentLexer(debug=0)
663 self.lexer = lexer
664 self.tokens = lexer.tokens
665 self.parser = yacc.yacc(module=self, start="file_input_end",
666 debug=debug, write_tables=False)
667
668 self.sd = create_pdecode()
669
670 def parse(self, code):
671 # self.lexer.input(code)
672 result = self.parser.parse(code, lexer=self.lexer, debug=self.debug)
673 return ast.Module(result)
674
675
676 ###### Code generation ######
677
678 #from compiler import misc, syntax, pycodegen
679
680 class GardenSnakeCompiler(object):
681 def __init__(self, debug=False):
682 self.parser = GardenSnakeParser(debug=debug)
683
684 def compile(self, code, mode="exec", filename="<string>"):
685 tree = self.parser.parse(code)
686 print("snake")
687 pprint(tree)
688 return tree
689 #misc.set_filename(filename, tree)
690 return compile(tree, mode="exec", filename="<string>")
691 # syntax.check(tree)
692 gen = pycodegen.ModuleCodeGenerator(tree)
693 code = gen.getCode()
694 return code