| 166,6 → 166,7 |
|---|
| "du", |
| ] |
| |
| |
| # Add kind flag to identifier in id2kind |
| def id_add_kind(identifier, kind): |
| if identifier not in id2kind: |
| 172,6 → 173,7 |
|---|
| id2kind[identifier] = '' |
| id2kind[identifier] += kind |
| |
| |
| # Remove kind flag of identifier in id2kind |
| def id_remove_kind(identifier, kind): |
| if identifier in id2kind: |
| 178,6 → 180,7 |
|---|
| if kind in id2kind[identifier]: |
| id2kind[identifier] = id2kind[identifier].replace(kind, '') |
| |
| |
| # Get kind of an identifier |
| def id_get_kind(identifier): |
| if identifier in id2kind: |
| 185,6 → 188,7 |
|---|
| else: |
| return '' |
| |
| |
| class LegacyAsmReader: |
| def __init__(self, file): |
| self.file = file |
| 196,8 → 200,10 |
|---|
| return self.lines[self.line_idx] |
| |
| def curr(self): |
| try: return self.lines[self.line_idx][self.i] |
| except: return '' |
| try: |
| return self.lines[self.line_idx][self.i] |
| except: |
| return '' |
| |
| def step(self): |
| c = self.curr() |
| 237,6 → 243,7 |
|---|
| while self.curr().isspace(): |
| self.step() |
| |
| |
| class AsmReaderRecognizingStrings(LegacyAsmReader): |
| def __init__(self, file): |
| super().__init__(file) |
| 247,9 → 254,9 |
|---|
| c = super().step() |
| if self.should_recognize_strings and (c == '"' or c == "'"): |
| # If just now we was at the double or single quotation mark |
| # and we aren't in a string yet |
| # then say "we are in a string openned with this quotation mark now" |
| if self.in_string == None: |
| # and we aren't in a string yet then say |
| # "we are in a string openned with this quotation mark now" |
| if self.in_string is None: |
| self.in_string = c |
| # If just now we was at the double or single quotation mark |
| # and we are in the string entered with the same quotation mark |
| 258,6 → 265,7 |
|---|
| self.in_string = None |
| return c |
| |
| |
| class AsmReaderReadingComments(AsmReaderRecognizingStrings): |
| def __init__(self, file): |
| super().__init__(file) |
| 282,8 → 290,11 |
|---|
| self.status_has_code = True |
| |
| def update_status(self): |
| # If we aren't in a comment and we aren't in a string - say we are now in a comment if ';' met |
| if not self.status_has_comment and not self.in_string and self.curr() == ';': |
| # If we aren't in a comment and we aren't in a string - |
| # say we are now in a comment if ';' met |
| if (not self.status_has_comment and |
| not self.in_string and |
| self.curr() == ';'): |
| self.status_set_has_comment() |
| # Else if we are in a comment - collect the comment |
| elif self.status_has_comment: |
| 305,7 → 316,8 |
|---|
| super().nextline() |
| # If the line we leave was not a comment-only line |
| # then forget the collected comment |
| # Otherwise the collected comment should be complemented by comment from next line in step() |
| # Otherwise the collected comment should be complemented by |
| # comment from next line in step() |
| if self.status_has_code: |
| # But we should preserve comment for the next line |
| # If previous line set align (cause many functions re documented |
| 314,9 → 326,11 |
|---|
| self.comment = '' |
| # Reset the line status (now it's the status of the new line) |
| self.status_reset() |
| # Set new status for this line according to the first character in the line |
| # Set new status for this line according to the |
| # first character in the line |
| self.update_status() |
| |
| |
| class AsmReaderFetchingIdentifiers(AsmReaderReadingComments): |
| def __init__(self, file): |
| super().__init__(file) |
| 328,10 → 342,12 |
|---|
| result += self.step() |
| return result |
| |
| |
| class AsmReader(AsmReaderFetchingIdentifiers): |
| def __init__(self, file): |
| super().__init__(file) |
| |
| |
| def append_file(full_path, contents): |
| if debug_mode: |
| if full_path not in output_files: |
| 342,11 → 358,13 |
|---|
| f.write(contents) |
| f.close() |
| |
| |
| class AsmElement: |
| def __init__(self, location, name, comment): |
| global warnings |
| |
| # If the element was constructed during this execution then the element is new |
| # If the element was constructed during this execution then |
| # the element is new |
| self.new = True |
| self.location = location |
| self.file = self.location.split(':')[0].replace('\\', '/') |
| 374,19 → 392,22 |
|---|
| if not doxycomment.endswith('\n'): |
| doxycomment += '\n' |
| if doxycomment.split('@brief ')[1][0].islower(): |
| warnings += f"{self.location}: Brief comment starting from lowercase\n" |
| warnings += (f"{self.location}: Brief comment starting from " + |
| "lowercase\n") |
| # Build contents to emit |
| contents = '' |
| contents += '/**\n' |
| contents += doxycomment |
| contents += (f"@par Source\n" + |
| f"<a href='{link_root}/{self.file}#line-{self.line}'>{self.file}:{self.line}</a>\n") |
| f"<a href='{link_root}/{self.file}" + |
| f"#line-{self.line}'>{self.file}:{self.line}</a>\n") |
| contents += '*/\n' |
| contents += declaration |
| contents += '\n\n' |
| # Get path to file to emit this |
| full_path = dest + '/' + self.file |
| # Remove the file on first access if it was created by previous generation |
| # Remove the file on first access if it was |
| # created by previous generation |
| if full_path not in created_files: |
| if os.path.isfile(full_path): |
| os.remove(full_path) |
| 397,6 → 418,7 |
|---|
| |
| append_file(full_path, contents) |
| |
| |
| class AsmVariable(AsmElement): |
| def __init__(self, location, name, comment, type, init): |
| super().__init__(location, name, comment) |
| 422,8 → 444,10 |
|---|
| # Emit this |
| super().emit(dest, doxycomment, declaration) |
| |
| |
| class AsmFunction(AsmElement): |
| def __init__(self, location, name, comment, calling_convention, args, used_regs): |
| def __init__(self, location, name, comment, calling_convention, |
| args, used_regs): |
| super().__init__(location, name, comment) |
| self.calling_convention = calling_convention |
| self.args = args |
| 473,6 → 497,7 |
|---|
| # Emit this |
| super().emit(dest, doxycomment, declaration) |
| |
| |
| class AsmLabel(AsmElement): |
| def __init__(self, location, name, comment): |
| super().__init__(location, name, comment) |
| 493,6 → 518,7 |
|---|
| # Emit this |
| super().emit(dest, doxycomment, declaration) |
| |
| |
| class AsmMacro(AsmElement): |
| def __init__(self, location, name, comment, args): |
| super().__init__(location, name, comment) |
| 526,6 → 552,7 |
|---|
| # Emit this |
| super().emit(dest, doxycomment, declaration) |
| |
| |
| class AsmStruct(AsmElement): |
| def __init__(self, location, name, comment, members): |
| super().__init__(location, name, comment) |
| 546,11 → 573,13 |
|---|
| declaration = f"struct {self.name}" + " {\n" |
| for member in self.members: |
| if type(member) == AsmVariable: |
| declaration += f'\t{member.type} {member.name}; /**< {member.comment} */\n' |
| declaration += (f'\t{member.type} {member.name}; ' + |
| f'/**< {member.comment} */\n') |
| declaration += '};' |
| # Emit this |
| super().emit(dest, doxycomment, declaration) |
| |
| |
| class AsmUnion(AsmElement): |
| def __init__(self, location, name, comment, members): |
| super().__init__(location, name, comment) |
| 571,16 → 600,20 |
|---|
| # Emit this |
| super().emit(dest, doxycomment, declaration) |
| |
| |
| class VariableNameIsMacroName: |
| def __init__(self, name): |
| self.name = name |
| |
| |
| def is_id(c): |
| return c.isprintable() and c not in "+-/*=<>()[]{};:,|&~#`'\" \n\r\t\v" |
| |
| |
| def is_starts_as_id(s): |
| return not s[0].isdigit() |
| |
| |
| def parse_after_macro(r): |
| location = r.location() |
| |
| 620,8 → 653,9 |
|---|
| r.step() |
| # Something unexpected |
| else: |
| raise Exception(f"Unexpected symbol '{r.curr()}' at index #{r.i} " + |
| f"in the macro declaration at {location} " + |
| raise Exception(f"Unexpected symbol '{r.curr()}' " + |
| f"at index #{r.i} in the macro declaration " + |
| f"at {location} " + |
| f"(line: {r.lines[r.line_idx]})\n''") |
| # Append the last argument |
| if arg != '': |
| 628,7 → 662,8 |
|---|
| args.append(arg) |
| # Skip t spaces after the argument list |
| r.skip_spaces() |
| # Get a comment if it is: read till the end of the line and get the comment from the reader |
| # Get a comment if it is: read till the end of the line and |
| # get the comment from the reader |
| while r.curr() != '': |
| r.step() |
| comment = r.comment |
| 645,6 → 680,7 |
|---|
| # Build the output |
| return AsmMacro(location, name, comment, args) |
| |
| |
| def parse_variable(r, first_word = None): |
| global warnings |
| location = r.location() |
| 654,7 → 690,7 |
|---|
| # Get variable name |
| name = "" |
| # Read it if it was not supplied |
| if first_word == None: |
| if first_word is None: |
| while is_id(r.curr()): |
| name += r.step() |
| # Or use the supplied one instead |
| 661,7 → 697,8 |
|---|
| else: |
| name = first_word |
| # Check the name |
| # If it's 0 len, that means threr's something else than an identifier at the beginning |
| # If it's 0 len, that means threr's something else than an |
| # identifier at the beginning |
| if len(name) == 0: |
| return None |
| # If it starts from digit or othervice illegally it's illegal |
| 675,7 → 712,8 |
|---|
| # If it's a macro name, that's not a variable declaration |
| if ID_KIND_MACRO_NAME in kind: |
| return VariableNameIsMacroName(name) |
| # If it's a datatype or a structure name that's not a variable declaration: that's just a data |
| # If it's a datatype or a structure name that's not a |
| # variable declaration: that's just a data |
| # don't document just a data for now |
| if ID_KIND_STRUCT_NAME in kind or ID_KIND_FASM_TYPE in kind: |
| return None |
| 714,6 → 752,7 |
|---|
| # Build the result |
| return AsmVariable(location, name, r.comment, var_type, value) |
| |
| |
| def parse_after_struct(r, as_union = True): |
| global warnings |
| location = r.location() |
| 747,7 → 786,8 |
|---|
| elif r.curr() == ':': |
| warnings += f"{r.location()}: Skept the label in the struct\n" |
| else: |
| raise Exception(f"Garbage in struct member at {location} (got '{var}' identifier)") |
| raise Exception(f"Garbage in struct member at {location} " + |
| f" (got '{var}' identifier)") |
| elif type(var) == VariableNameIsMacroName: |
| if var.name == 'ends': |
| break |
| 758,6 → 798,7 |
|---|
| else: |
| return AsmUnion(location, name, comment, members) |
| |
| |
| def parse_after_proc(r): |
| # Get proc name |
| name = r.fetch_identifier() |
| 813,8 → 854,10 |
|---|
| r.step() |
| comment = r.comment |
| # Build the element |
| return AsmFunction(r.location(), name, comment, calling_convention, args, used_regs) |
| return AsmFunction(r.location(), name, comment, calling_convention, |
| args, used_regs) |
| |
| |
| def get_declarations(asm_file_contents, asm_file_name): |
| r = AsmReader(asm_file_name) |
| |
| 857,7 → 900,8 |
|---|
| pass |
| elif first_word == 'purge': |
| while True: |
| # Skip spaces after the 'purge' keyword or after the comma what separated the previous macro name |
| # Skip spaces after the 'purge' keyword or after |
| # the comma what separated the previous macro name |
| r.skip_spaces() |
| # Get the purged macro name |
| name = '' |
| 870,7 → 914,8 |
|---|
| pass |
| # Skip spaces after the name |
| r.skip_spaces() |
| # If it's comma (',') after then that's not the last purged macro, continue purging |
| # If it's comma (',') after then that's not the last purged |
| # macro, continue purging |
| if r.curr() == ',': |
| r.step() |
| continue |
| 893,14 → 938,19 |
|---|
| name = var |
| # Match label beginning (':' after name) |
| if r.curr() == ':': |
| # Get to the end of the line and get the coment from the reader |
| # Get to the end of the line and |
| # get the coment from the reader |
| while r.curr() != '': |
| r.step() |
| comment = r.comment |
| # Only handle non-local labels |
| if name[0] != '.' and name != "@@" and name != "$Revision": |
| # Treate the label as function if there's @return or |
| # @param in its comment. Othervice it's just a variable |
| # with type `label` in generated doxygen C |
| if '@return' in comment or '@param' in comment: |
| element = AsmFunction(r.location(), name, comment, '', [], []) |
| element = AsmFunction(r.location(), name, comment, |
| '', [], []) |
| else: |
| element = AsmLabel(r.location(), name, comment) |
| elements.append(element) |
| 914,6 → 964,7 |
|---|
| id_add_kind(word_one, ID_KIND_EQUATED_CONSTANT) |
| r.nextline() |
| |
| |
| def it_neds_to_be_parsed(source_file): |
| # If there's no symbols file saved - parse it anyway |
| # cause we need to create the symbols file and use it |
| 933,6 → 984,7 |
|---|
| return True |
| return False |
| |
| |
| def handle_file(handled_files, asm_file_name, subdir = "."): |
| global elements |
| # Canonicalize the file path and get it relative to cwd |
| 947,7 → 999,8 |
|---|
| return |
| # Say that the file was handled in this execution |
| handled_files.append(asm_file_name) |
| # Check if the file should be parsed (if it was modified or wasn't parsed yet) |
| # Check if the file should be parsed |
| # (if it was modified or wasn't parsed yet) |
| should_get_declarations = True |
| if not it_neds_to_be_parsed(asm_file_name): |
| print(f"Skipping {asm_file_name} (already newest)") |
| 955,8 → 1008,12 |
|---|
| else: |
| print(f"Handling {asm_file_name}") |
| # Remove elements parsed from this file before if any |
| elements_to_remove = [x for x in elements if x.location.split(':')[0] == asm_file_name] |
| elements = [x for x in elements if x.location.split(':')[0] != asm_file_name] |
| elements_to_remove = [ |
| x for x in elements if x.location.split(':')[0] == asm_file_name |
| ] |
| elements = [ |
| x for x in elements if x.location.split(':')[0] != asm_file_name |
| ] |
| # Forget types of identifiers of names of the removed elements |
| for element in elements_to_remove: |
| if type(element) == AsmStruct: |
| 966,10 → 1023,11 |
|---|
| # Read the source |
| asm_file_contents = open(asm_file_name, "r", encoding="utf-8").read() |
| # Find includes, fix their paths and handle em recoursively |
| includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents, flags=re.MULTILINE) |
| includes = re.findall(r'^include (["\'])(.*)\1', asm_file_contents, |
| flags=re.MULTILINE) |
| for include in includes: |
| include = include[1].replace('\\', '/'); |
| full_path = subdir + '/' + include; |
| include = include[1].replace('\\', '/') |
| full_path = subdir + '/' + include |
| # If the path isn't valid, maybe that's not relative path |
| if not os.path.isfile(full_path): |
| full_path = include |
| 980,7 → 1038,8 |
|---|
| get_declarations(asm_file_contents, asm_file_name) |
| |
| if __name__ == "__main__": |
| link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk" |
| link_root = "http://websvn.kolibrios.org/filedetails.php" |
| link_root += "?repname=Kolibri+OS&path=/kernel/trunk" |
| |
| # Dict where an identifier is assicoated with a string |
| # The string contains characters specifying flags |
| 1023,12 → 1082,24 |
|---|
| # Parse arguments |
| parser = argparse.ArgumentParser() |
| parser.add_argument("-o", help="Doxygen output folder") |
| parser.add_argument("--clean", help="Remove generated files", action="store_true") |
| parser.add_argument("--dump", help="Dump all defined symbols", action="store_true") |
| parser.add_argument("--stats", help="Print symbol stats", action="store_true") |
| parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true") |
| parser.add_argument("--noemit", help="Do not emit doxygen files (for testing)", action="store_true") |
| parser.add_argument("--debug", help="Show hashes of files (for testing)", action="store_true") |
| parser.add_argument("--clean", |
| help="Remove generated files", |
| action="store_true") |
| parser.add_argument("--dump", |
| help="Dump all defined symbols", |
| action="store_true") |
| parser.add_argument("--stats", |
| help="Print symbol stats", |
| action="store_true") |
| parser.add_argument("--nowarn", |
| help="Do not write warnings file", |
| action="store_true") |
| parser.add_argument("--noemit", |
| help="Do not emit doxygen files (for testing)", |
| action="store_true") |
| parser.add_argument("--debug", |
| help="Show hashes of files (for testing)", |
| action="store_true") |
| args = parser.parse_args() |
| doxygen_src_path = args.o if args.o else 'docs/doxygen' |
| clean_generated_stuff = args.clean |
| 1047,9 → 1118,11 |
|---|
| # Load remembered list of symbols |
| if os.path.isfile('asmxygen.elements.pickle'): |
| print('Reading existing dump of symbols') |
| (elements, id2kind) = pickle.load(open('asmxygen.elements.pickle', 'rb')) |
| pickle_file = open('asmxygen.elements.pickle', 'rb') |
| (elements, id2kind) = pickle.load(pickle_file) |
| pickle_file.close() |
| |
| handle_file(kernel_files, "./kernel.asm"); |
| handle_file(kernel_files, "./kernel.asm") |
| |
| if dump_symbols: |
| stdout = sys.stdout |
| 1072,16 → 1145,20 |
|---|
| i = 0 |
| new_elements = [x for x in elements if x.new] |
| for element in new_elements: |
| print(f"[{i + 1}/{len(new_elements)}] Emitting {element.name} from {element.location}") |
| counter = f"[{i + 1}/{len(new_elements)}]" |
| print(f"{counter} Emitting {element.name} from {element.location}") |
| element.emit(doxygen_src_path) |
| i += 1 |
| |
| print(f"Writing dump of symbols to asmxygen.elements.pickle") |
| |
| # Now when the new elements already was written, there's no new elements anymore |
| # Now when the new elements already was written, there's no new |
| # elements anymore |
| for element in elements: |
| element.new = False |
| pickle.dump((elements, id2kind), open('asmxygen.elements.pickle', 'wb')) |
| pickle_file = open('asmxygen.elements.pickle', 'wb') |
| pickle.dump((elements, id2kind), pickle_file) |
| pickle_file.close() |
| |
| if print_stats: |
| var_count = 0 |
| 1124,6 → 1201,7 |
|---|
| else: |
| reference_hash_per_file = open("asmxygen_hash_per_file.txt").read() |
| if reference_hash_per_file != hash_per_file: |
| print(''.join(difflib.ndiff(reference_hash_per_file, hash_per_file))) |
| diffs = difflib.ndiff(reference_hash_per_file, hash_per_file) |
| print(''.join(diffs)) |
| else: |
| print("SUCCESS") |