WebSVN – Kolibri OS – Diff – /kernel/trunk/asmxygen.py

 import hashlib
 import difflib
 # fasm keywords
     "xchg", "xend", "xgetbv", "xlat", "xlatb", "xor", "xorpd", "xorps",
     "xrstor", "xrstors", "xsave", "xsavec", "xsaveopt", "xsaves", "xsetbv",
+]
     "dq", "rq",
     "dt", "rt",
     "du",
+]
 # Add kind flag to identifier in id2kind
 def id_add_kind(identifier, kind):
     if identifier not in id2kind:
         id2kind[identifier] = ''
+    id2kind[identifier] += kind
     id2kind[identifier] += kind
 # Remove kind flag of identifier in id2kind
 def id_remove_kind(identifier, kind):
     if identifier in id2kind:
+        if kind in id2kind[identifier]:
-        if kind in id2kind[identifier]:
+            id2kind[identifier] = id2kind[identifier].replace(kind, '')
             id2kind[identifier] = id2kind[identifier].replace(kind, '')
 # Get kind of an identifier
 def id_get_kind(identifier):
     if identifier in id2kind:
+        return id2kind[identifier]
-        return id2kind[identifier]
+    else:
-    else:
+        return ''
         return ''
 class LegacyAsmReader:
     def currline(self):
         return self.lines[self.line_idx]
+    def curr(self):
+        try:
-    def curr(self):
+            return self.lines[self.line_idx][self.i]
-        try: return self.lines[self.line_idx][self.i]
+        except:
-        except: return ''
+            return ''
     def step(self):
     def no_lines(self):
         if self.line_idx >= len(self.lines):
             return True
         return False
     def location(self):
         return f"{self.file}:{self.line_idx + 1}"
     def skip_spaces(self):
+        while self.curr().isspace():
-        while self.curr().isspace():
+            self.step()
             self.step()
 class AsmReaderRecognizingStrings(LegacyAsmReader):
     def __init__(self, file):
         super().__init__(file)
         self.in_string = None
         self.should_recognize_strings = True
     def step(self):
         c = super().step()
         if self.should_recognize_strings and (c == '"' or c == "'"):
             # If just now we was at the double or single quotation mark
-            # and we aren't in a string yet
+            # and we aren't in a string yet then say
-            # then say "we are in a string openned with this quotation mark now"
+            # "we are in a string openned with this quotation mark now"
-            if self.in_string == None:
+            if self.in_string is None:
                 self.in_string = c
             # If just now we was at the double or single quotation mark
             # and we are in the string entered with the same quotation mark
+            # then say "we aren't in a string anymore"
-            # then say "we aren't in a string anymore"
+            elif self.in_string == c:
-            elif self.in_string == c:
+                self.in_string = None
-                self.in_string = None
+        return c
         return c
     def status_set_has_code(self):
         self.status_has_code = True
+    def update_status(self):
-    def update_status(self):
+        # If we aren't in a comment and we aren't in a string -
+        # say we are now in a comment if ';' met
+        if (not self.status_has_comment and
-        # If we aren't in a comment and we aren't in a string - say we are now in a comment if ';' met
+            not self.in_string and
-        if not self.status_has_comment and not self.in_string and self.curr() == ';':
+            self.curr() == ';'):
             self.status_set_has_comment()
         # Else if we are in a comment - collect the comment
         elif self.status_has_comment:
     def nextline(self):
         prev_line = self.currline()
         super().nextline()
         # If the line we leave was not a comment-only line
         # then forget the collected comment
-        # Otherwise the collected comment should be complemented by comment from next line in step()
+        # Otherwise the collected comment should be complemented by
+        # comment from next line in step()
         if self.status_has_code:
             # But we should preserve comment for the next line
             # If previous line set align (cause many functions re documented
             # right before align set, not before their labels)
             if not prev_line.startswith("align "):
                 self.comment = ''
         # Reset the line status (now it's the status of the new line)
         self.status_reset()
-        # Set new status for this line according to the first character in the line
+        # Set new status for this line according to the
+        # first character in the line
         self.update_status()
 class AsmReaderFetchingIdentifiers(AsmReaderReadingComments):
     def __init__(self, file):
         super().__init__(file)
         result = ''
         while is_id(self.curr()):
             result += self.step()
         return result
 class AsmReader(AsmReaderFetchingIdentifiers):
     def __init__(self, file):
+        super().__init__(file)
         super().__init__(file)
 def append_file(full_path, contents):
     if debug_mode:
         if full_path not in output_files:
             output_files[full_path] = ""
         output_files[full_path] += contents
     else:
         f = open(full_path, "a")
+        f.write(contents)
-        f.write(contents)
+        f.close()
         f.close()
 class AsmElement:
+    def __init__(self, location, name, comment):
-    def __init__(self, location, name, comment):
+        global warnings
         global warnings
         # If the element was constructed during this execution then
-        # If the element was constructed during this execution then the element is new
+        # the element is new
         self.new = True
     def dump(self):
         print(f"\n{self.location}: {self.name}")
         print(f"{self.comment}")
-    def emit(self, dest, doxycomment = '', declaration = ''):
+    def emit(self, dest, doxycomment='', declaration=''):
         # Do not emit anything if the symbol is marked as hidden in its comment
         if '@dont_give_a_doxygen' in self.comment:
             declaration = f'#define {self.name}'
         # Check doxycomment
         if not doxycomment.endswith('\n'):
             doxycomment += '\n'
         if doxycomment.split('@brief ')[1][0].islower():
-            warnings += f"{self.location}: Brief comment starting from lowercase\n"
+            warnings += (f"{self.location}: Brief comment starting from " +
+                         "lowercase\n")
         # Build contents to emit
         contents = ''
         contents += '/**\n'
         contents += doxycomment
         contents += (f"@par Source\n" +
+                     f"#line-{self.line}'>{self.file}:{self.line}\n")
         contents += '*/\n'
         contents += declaration
         contents += '\n\n'
         # Get path to file to emit this
         full_path = dest + '/' + self.file
-        # Remove the file on first access if it was created by previous generation
+        # Remove the file on first access if it was
+        # created by previous generation
         if full_path not in created_files:
             if os.path.isfile(full_path):
                 os.remove(full_path)
             created_files.append(full_path)
         # Create directories need for the file
         os.makedirs(os.path.dirname(full_path), exist_ok=True)
         contents = ''.join([i if ord(i) < 128 else '?' for i in contents])
+        append_file(full_path, contents)
         append_file(full_path, contents)
 class AsmVariable(AsmElement):
     def __init__(self, location, name, comment, type, init):
         super().__init__(location, name, comment)
         var_type = self.type.replace(".", "_")
         declaration = f"{var_type} {name};"
         # Emit this
         super().emit(dest, doxycomment, declaration)
 class AsmFunction(AsmElement):
+    def __init__(self, location, name, comment, calling_convention,
-    def __init__(self, location, name, comment, calling_convention, args, used_regs):
+                 args, used_regs):
         super().__init__(location, name, comment)
         self.calling_convention = calling_convention
         self.args = args
         name = self.name.replace(".", "_")
         declaration = f"void {name}{arg_list};"
         # Emit this
         super().emit(dest, doxycomment, declaration)
 class AsmLabel(AsmElement):
     def __init__(self, location, name, comment):
         super().__init__(location, name, comment)
         name = self.name.replace(".", "_")
         declaration = f"label {name};"
         # Emit this
         super().emit(dest, doxycomment, declaration)
 class AsmMacro(AsmElement):
     def __init__(self, location, name, comment, args):
         super().__init__(location, name, comment)
         # Build declaration
         declaration = f"#define {self.name}{arg_list}"
         # Emit this
         super().emit(dest, doxycomment, declaration)
 class AsmStruct(AsmElement):
     def __init__(self, location, name, comment, members):
         super().__init__(location, name, comment)
         doxycomment += '\n'
         # Build declaration
         declaration = f"struct {self.name}" + " {\n"
         for member in self.members:
             if type(member) == AsmVariable:
-                declaration += f'\t{member.type} {member.name}; /**< {member.comment} */\n'
+                declaration += (f'\t{member.type} {member.name}; ' +
+                                f'/**< {member.comment} */\n')
         declaration += '};'
         # Emit this
         super().emit(dest, doxycomment, declaration)
 class AsmUnion(AsmElement):
     def __init__(self, location, name, comment, members):
         super().__init__(location, name, comment)
         # Build declaration
         declaration = f"union {self.name}" + " {};"
         # Emit this
         super().emit(dest, doxycomment, declaration)
 class VariableNameIsMacroName:
     def __init__(self, name):
+        self.name = name
         self.name = name
+def is_id(c):
-def is_id(c):
+    return c.isprintable() and c not in "+-/*=<>()[]{};:,|&~#`'\" \n\r\t\v"
     return c.isprintable() and c not in "+-/*=<>()[]{};:,|&~#`'\" \n\r\t\v"
 def is_starts_as_id(s):
-def is_starts_as_id(s):
+    return not s[0].isdigit()
     return not s[0].isdigit()
         # Just skip whitespaces
         elif r.curr().isspace():
             r.step()
         # Something unexpected
         else:
-            raise Exception(f"Unexpected symbol '{r.curr()}' at index #{r.i} " +
+            raise Exception(f"Unexpected symbol '{r.curr()}' " +
-                            f"in the macro declaration at {location} " +
+                            f"at index #{r.i} in the macro declaration " +
+                            f"at {location} " +
                             f"(line: {r.lines[r.line_idx]})\n''")
     # Append the last argument
     if arg != '':
         args.append(arg)
     # Skip t spaces after the argument list
     r.skip_spaces()
-    # Get a comment if it is: read till the end of the line and get the comment from the reader
+    # Get a comment if it is: read till the end of the line and
+    # get the comment from the reader
     while r.curr() != '':
         r.step()
     comment = r.comment
     # Find end of the macro
     prev = ''
             continue
         prev = r.step()
     # Build the output
     return AsmMacro(location, name, comment, args)
-def parse_variable(r, first_word = None):
+def parse_variable(r, first_word=None):
     global warnings
     location = r.location()
     # Skip spaces before variable name
     r.skip_spaces()
     # Get variable name
     name = ""
     # Read it if it was not supplied
-    if first_word == None:
+    if first_word is None:
         while is_id(r.curr()):
             name += r.step()
     # Or use the supplied one instead
     else:
         name = first_word
+    # Check the name
-    # Check the name
+    # If it's 0 len, that means threr's something else than an
-    # If it's 0 len, that means threr's something else than an identifier at the beginning
+    # identifier at the beginning
     if len(name) == 0:
         return None
     # If it starts from digit or othervice illegally it's illegal
     if ID_KIND_KEYWORD in kind:
         return None
     # If it's a macro name, that's not a variable declaration
     if ID_KIND_MACRO_NAME in kind:
         return VariableNameIsMacroName(name)
-    # If it's a datatype or a structure name that's not a variable declaration: that's just a data
+    # If it's a datatype or a structure name that's not a
+    # variable declaration: that's just a data
     # don't document just a data for now
     if ID_KIND_STRUCT_NAME in kind or ID_KIND_FASM_TYPE in kind:
         return None
     # Skip spaces before type name
     r.skip_spaces()
     while r.curr() != '':
         r.step()
     # Build the result
     return AsmVariable(location, name, r.comment, var_type, value)
-def parse_after_struct(r, as_union = True):
+def parse_after_struct(r, as_union=True):
     global warnings
     location = r.location()
         if type(var) == AsmVariable:
             members.append(var)
         elif type(var) == str:
             if var == 'union':
                 # Parse the union as a struct
-                union = parse_after_struct(r, as_union = True)
+                union = parse_after_struct(r, as_union=True)
                 members.append(union)
                 # Skip the ends of the union
                 r.nextline()
             elif r.curr() == ':':
                 warnings += f"{r.location()}: Skept the label in the struct\n"
             else:
-                raise Exception(f"Garbage in struct member at {location} (got '{var}' identifier)")
+                raise Exception(f"Garbage in struct member at {location} " +
+                                f" (got '{var}' identifier)")
         elif type(var) == VariableNameIsMacroName:
             if var.name == 'ends':
                 break
         r.nextline()
     # Return the result
     if as_union:
         return AsmStruct(location, name, comment, members)
     else:
         return AsmUnion(location, name, comment, members)
 def parse_after_proc(r):
     # Get proc name
     name = r.fetch_identifier()
     # Next identifier after the proc name
     # Get to the end of the line and get a comment from the reader
     while r.curr() != '':
         r.step()
     comment = r.comment
     # Build the element
-    return AsmFunction(r.location(), name, comment, calling_convention, args, used_regs)
+    return AsmFunction(r.location(), name, comment, calling_convention,
+                       args, used_regs)
 def get_declarations(asm_file_contents, asm_file_name):
     r = AsmReader(asm_file_name)
         elif first_word == 'repeat':
             # Skip the repeat directive
             pass
         elif first_word == 'purge':
             while True:
+                # Skip spaces after the 'purge' keyword or after
-                # Skip spaces after the 'purge' keyword or after the comma what separated the previous macro name
+                # the comma what separated the previous macro name
                 r.skip_spaces()
                 # Get the purged macro name
                 name = ''
                 while is_id(r.curr()):
                     name += r.step()
                     id_remove_kind(name, ID_KIND_MACRO_NAME)
                 except:
                     pass
                 # Skip spaces after the name
                 r.skip_spaces()
-                # If it's comma (',') after then that's not the last purged macro, continue purging
+                # If it's comma (',') after then that's not the last purged
+                # macro, continue purging
                 if r.curr() == ',':
                     r.step()
                     continue
                 # Here we purged all the macros should be purged
                 break
             # If it didn't match a type identifier after the word
             elif type(var) == str:
                 name = var
                 # Match label beginning (':' after name)
                 if r.curr() == ':':
+                    # Get to the end of the line and
-                    # Get to the end of the line and get the coment from the reader
+                    # get the coment from the reader
                     while r.curr() != '':
                         r.step()
                     comment = r.comment
                     # Only handle non-local labels
                     if name[0] != '.' and name != "@@" and name != "$Revision":
+                        # Treate the label as function if there's @return or
+                        # @param in its comment. Othervice it's just a variable
+                        # with type `label` in generated doxygen C
                         if '@return' in comment or '@param' in comment:
-                            element = AsmFunction(r.location(), name, comment, '', [], [])
+                            element = AsmFunction(r.location(), name, comment,
+                                                  '', [], [])
                         else:
                             element = AsmLabel(r.location(), name, comment)
                         elements.append(element)
                 elif r.curr() == '=':
                     # Save the identifier as a set constant
                 if word_two == 'equ':
                     # Save the identifier as an equated constant
                     id_add_kind(word_one, ID_KIND_EQUATED_CONSTANT)
         r.nextline()
 def it_neds_to_be_parsed(source_file):
     # If there's no symbols file saved - parse it anyway
     # cause we need to create the symbols file and use it
     # if we gonna generate proper doxygen
     # then the source should be recompiled (existing doxygen is old)
     if source_change_time > dest_change_file:
         return True
     return False
-def handle_file(handled_files, asm_file_name, subdir = "."):
+def handle_file(handled_files, asm_file_name, subdir="."):
     global elements
     # Canonicalize the file path and get it relative to cwd
     cwd = os.path.abspath(os.path.dirname(sys.argv[0]))
     asm_file_name = os.path.realpath(asm_file_name)
     # If the file was handled in this execution before - skip it
     if asm_file_name in handled_files:
         return
     # Say that the file was handled in this execution
     handled_files.append(asm_file_name)
+    # Check if the file should be parsed
-    # Check if the file should be parsed (if it was modified or wasn't parsed yet)
+    # (if it was modified or wasn't parsed yet)
     should_get_declarations = True
     if not it_neds_to_be_parsed(asm_file_name):
         print(f"Skipping {asm_file_name} (already newest)")
         should_get_declarations = False
     else:
         print(f"Handling {asm_file_name}")
         # Remove elements parsed from this file before if any
+        elements_to_remove = [
-        elements_to_remove = [x for x in elements if x.location.split(':')[0] == asm_file_name]
+            x for x in elements if x.location.split(':')[0] == asm_file_name
+        ]
+        elements = [
-        elements = [x for x in elements if x.location.split(':')[0] != asm_file_name]
+            x for x in elements if x.location.split(':')[0] != asm_file_name
+        ]
         # Forget types of identifiers of names of the removed elements
         for element in elements_to_remove:
             if type(element) == AsmStruct:
                 id_remove_kind(element.name, ID_KIND_STRUCT_NAME)
             elif type(element) == AsmMacro:
                 id_remove_kind(element.name, ID_KIND_MACRO_NAME)
     # Read the source
     asm_file_contents = open(asm_file_name, "r", encoding="utf-8").read()
     # Find includes, fix their paths and handle em recoursively
-    includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents, flags=re.MULTILINE)
+    includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents,
+                          flags=re.MULTILINE)
     for include in includes:
-        include = include[1].replace('\\', '/');
+        include = include[1].replace('\\', '/')
-        full_path = subdir + '/' + include;
+        full_path = subdir + '/' + include
         # If the path isn't valid, maybe that's not relative path
         if not os.path.isfile(full_path):
             full_path = include
         new_subdir = full_path.rsplit('/', 1)[0]
         handle_file(handled_files, full_path, new_subdir)
     # Only collect declarations from the file if it wasn't parsed before
     if should_get_declarations and not clean_generated_stuff:
         get_declarations(asm_file_contents, asm_file_name)
 if __name__ == "__main__":
+    link_root = "http://websvn.kolibrios.org/filedetails.php"
-    link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk"
+    link_root += "?repname=Kolibri+OS&path=/kernel/trunk"
     # Dict where an identifier is assicoated with a string
     # The string contains characters specifying flags
     enable_warnings = True
     # Parse arguments
     parser = argparse.ArgumentParser()
+    parser.add_argument("-o", help="Doxygen output folder")
-    parser.add_argument("-o", help="Doxygen output folder")
+    parser.add_argument("--clean",
+                        help="Remove generated files",
+                        action="store_true")
-    parser.add_argument("--clean", help="Remove generated files", action="store_true")
+    parser.add_argument("--dump",
+                        help="Dump all defined symbols",
+                        action="store_true")
-    parser.add_argument("--dump", help="Dump all defined symbols", action="store_true")
+    parser.add_argument("--stats",
+                        help="Print symbol stats",
+                        action="store_true")
-    parser.add_argument("--stats", help="Print symbol stats", action="store_true")
+    parser.add_argument("--nowarn",
+                        help="Do not write warnings file",
+                        action="store_true")
-    parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true")
+    parser.add_argument("--noemit",
+                        help="Do not emit doxygen files (for testing)",
+                        action="store_true")
-    parser.add_argument("--noemit", help="Do not emit doxygen files (for testing)", action="store_true")
+    parser.add_argument("--debug",
+                        help="Show hashes of files (for testing)",
-    parser.add_argument("--debug", help="Show hashes of files (for testing)", action="store_true")
+                        action="store_true")
     args = parser.parse_args()
     doxygen_src_path = args.o if args.o else 'docs/doxygen'
     clean_generated_stuff = args.clean
     dump_symbols = args.dump
     # Variables, functions, labels, macros, structure types
     elements = []
     created_files = []
     kernel_files = []
-    output_files = {} # If --debug then all the files are written here
+    output_files = {}  # If --debug then all the files are written here
     # Load remembered list of symbols
     if os.path.isfile('asmxygen.elements.pickle'):
+        print('Reading existing dump of symbols')
+        pickle_file = open('asmxygen.elements.pickle', 'rb')
-        print('Reading existing dump of symbols')
+        (elements, id2kind) = pickle.load(pickle_file)
-        (elements, id2kind) = pickle.load(open('asmxygen.elements.pickle', 'rb'))
+        pickle_file.close()
-    handle_file(kernel_files, "./kernel.asm");
+    handle_file(kernel_files, "./kernel.asm")
     if dump_symbols:
         stdout = sys.stdout
-        sys.stdout = open('asmxygen.dump.txt', 'w', encoding = 'utf-8')
+        sys.stdout = open('asmxygen.dump.txt', 'w', encoding='utf-8')
         for asm_element in elements:
             asm_element.dump()
         sys.stdout = stdout
     if clean_generated_stuff:
         kernel_files_set = set(kernel_files)
         for file in kernel_files:
             doxygen_file = f"{doxygen_src_path}/{file}"
             if (os.path.isfile(doxygen_file)):
-                print(f"Removing {file}... ", end = '')
+                print(f"Removing {file}... ", end='')
                 os.remove(doxygen_file)
                 print("Done.")
+    elif not noemit:
-    elif not noemit:
+        print(f"Writing doumented sources to {doxygen_src_path}")
         print(f"Writing doumented sources to {doxygen_src_path}")
         i = 0
-        i = 0
+        new_elements = [x for x in elements if x.new]
-        new_elements = [x for x in elements if x.new]
+        for element in new_elements:
+            counter = f"[{i + 1}/{len(new_elements)}]"
-        for element in new_elements:
+            print(f"{counter} Emitting {element.name} from {element.location}")
-            print(f"[{i + 1}/{len(new_elements)}] Emitting {element.name} from {element.location}")
+            element.emit(doxygen_src_path)
-            element.emit(doxygen_src_path)
+            i += 1
+        print(f"Writing dump of symbols to asmxygen.elements.pickle")
             i += 1
         # Now when the new elements already was written, there's no new
-        print(f"Writing dump of symbols to asmxygen.elements.pickle")
+        # elements anymore
         for element in elements:
         print(f'Parsed function count: {fun_count}')
         print(f'Parsed union type count: {uni_count}')
         print(f'Parsed structure type count: {str_count}')
     if enable_warnings:
-        open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)
+        open('asmxygen.txt', "w", encoding="utf-8").write(warnings)
     if debug_mode:
         hash_per_file = ""
             open("asmxygen_hash_per_file.txt", "w").write(hash_per_file)
             print("NEW")
         else:
             reference_hash_per_file = open("asmxygen_hash_per_file.txt").read()
             if reference_hash_per_file != hash_per_file:
-                print(''.join(difflib.ndiff(reference_hash_per_file, hash_per_file)))
+                diffs = difflib.ndiff(reference_hash_per_file, hash_per_file)
+                print(''.join(diffs))
             else:
                 print("SUCCESS")

Subversion Repositories Kolibri OS

(root)/kernel/trunk/asmxygen.py @ 8977 – Rev 9407 → 9408