Subversion Repositories Kolibri OS

import re

import os

import argparse

# Parameters

# Path to doxygen folder to make doxygen files in: -o 

doxygen_src_path = 'docs/doxygen'

# Remove generated doxygen files: --clean

clean_generated_stuff = False

# Dump all defined symbols: --dump

dump_symbols = False

# Print symbol stats: --stats

print_stats = False

# Do not write warnings file: --nowarn

enable_warnings = True

# Constants

link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk"

# Warning list

warnings = ""

# Parse arguments

parser = argparse.ArgumentParser()

parser.add_argument("-o", help="Doxygen output folder")

parser.add_argument("--clean", help="Remove generated files", action="store_true")

parser.add_argument("--dump", help="Dump all defined symbols", action="store_true")

parser.add_argument("--stats", help="Print symbol stats", action="store_true")

parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true")

args = parser.parse_args()

doxygen_src_path = args.o if args.o else 'docs/doxygen'

clean_generated_stuff = args.clean

dump_symbols = args.dump

print_stats = args.stats

enable_warnings = not args.nowarn

# kernel_structure["filename"] = {

#   [ [],    # [0] Variables - [ line, name ]

#     [],    # [1] Macros - [ line, name ]

#     [],    # [2] Procedures - [ line, name ]

#     [],    # [3] Labels - [ line, name ]

#     [] ] } # [4] Structures - [ line, name ]

VARIABLES = 0

MACROS = 1

PROCEDURES = 2

LABELS = 3

STRUCTURES = 4

kernel_structure = {}

class AsmVariable:

	def __init__(self, line, name, type, init, comment, line_span):

		self.line = line

		self.name = name

		self.type = type

		self.init = init

		self.comment = comment # Comment after the definition (a dd 0 ; Comment)

		self.line_span = line_span # How much .asm lines its definition takes

class AsmFunction:

	def __init__(self, line, name):

		self.line = line

		self.name = name

class AsmLabel:

	def __init__(self, line, name):

		self.line = line

		self.name = name

class AsmMacro:

	def __init__(self, line, name, comment, args):

		self.line = line

		self.name = name

		self.comment = comment

		self.args = args

class AsmStruct:

	def __init__(self, line, name):

		self.line = line

		self.name = name

def parse_variable(asm_file_name, lines, line_idx):

	global warnings

	def curr():

		try: return line[i]

		except: return ''

	# Returns current and then increments current index

	def step():

		nonlocal i

		c = curr()

		i += 1

		return c

	line = lines[line_idx]

	i = 0

	# Skip first spaces

	while curr().isspace(): step()

	# Get name

	name = ""

	while curr().isalnum() or curr() == '_' or curr() == '.': name += step()

	# Skip spaces after variable name

	while curr().isspace(): step()

	# Get type specifier (db, dd, etc.)

	type = ""

	while curr().isalnum() or curr() == '_': type += step()

	# Skip spaces after type specifier

	while curr().isspace(): step()

	# Get initial value (everything up to end of the line or comment)

	init = ""

	while curr() and curr() != ';': init += step()

	# Get comment

	comment = ""

	if curr() == ';':

		step() # Skip ';'

		while curr(): comment += step()

	# Process type

	if type == "db": type = "byte"

	elif type == "dw": type = "word"

	elif type == "dd": type = "dword"

	elif type == "dq": type = "qword"

	else: raise Exception(f"Unexpected type: '{type}' (i = {i})")

	# Process comment

	if comment == "": comment = "Undocumented"

	else:

		comment = comment.lstrip()

		if (len(comment) == 0):

			comment = "!!! EMPTY_COMMENT"

			warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"

		if comment[0].islower():

			warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"

	# Build the result

	result = AsmVariable(line_idx + 1, name, type, init, comment, 1)

	return (1, result)

def is_id(c):

	return c.isprintable() and c not in "+-/*=<>()[]{}:,|&~#`'\" \n\r\t\v"

def get_comment_begin(line):

	result = len(line)

	in_str = False

	for i in range(len(line)):

		if in_str:

			if line[i] == in_str: in_str = False

			i += 1

		elif line[i] == '\'' or line[i] == '\"':

			in_str = line[i]

			i += 1

		elif line[i] == ';':

			result = i

			break

		else:

			i += 1

	return result

def get_comment(line):

	return line[get_comment_begin(line):]

def remove_comment(line):

	return line[0:get_comment_begin(line)]

def insert_comment(line, comment):

	comment_begin = get_comment_begin(line)

	line_left = line[:get_comment_begin(line)]

	line_right = line[get_comment_begin(line):]

	return line_left + comment + line_right

def has_line_wrap(line):

	if remove_comment(line).rstrip()[-1] == '\\':

		return True

	return False

def remove_line_wrap(line):

	if remove_comment(line).rstrip()[-1] == '\\':

		return remove_comment(line).rstrip()[:-1]

	return line

def parse_macro(asm_file_name, lines, line_idx):

	line_idx_orig = line_idx

	global warnings

	def curr():

		try: return line[i]

		except: return ''

	# Returns current and then increments current index

	def step():

		nonlocal i

		c = curr()

		i += 1

		return c

	line = lines[line_idx]

	# Handle line wraps ('\' at the end)

	while has_line_wrap(line):

		next_line = lines[line_idx + 1]

		prev_line_comment = get_comment(line)

		line = remove_line_wrap(line) + insert_comment(next_line, prev_line_comment)

		line_idx += 1

	i = 0

	# Skip first spaces

	while curr().isspace(): step()

	# Read "macro" keyword

	keyword = ""

	while is_id(curr()): keyword += step()

	if keyword != "macro": raise Exception(f"Not a macro: {line}")

	# Skip spaces after "macro"

	while curr().isspace(): step()

	# Read macro name

	name = ""

	while curr() and not curr().isspace(): name += step()

	# Skip spaces after macro name

	while curr().isspace(): step()

	# Find all arguments

	args = []

	arg = ''

	while curr() and curr() != ';' and curr() != '{':

		# Collect identifier

		if is_id(curr()):

			arg += step()

		# Save the collected identifier

		elif curr() == ',':

			args.append(arg)

			arg = ''

			step()

		# Just push the '['

		elif curr() == '[':

			args.append(step())

		# Just push the identifier and get ']' ready to be pushed on next comma

		elif curr() == ']':

			args.append(arg)

			arg = step()

		# Just push the identifier and get '*' ready to be pushed on next comma

		elif curr() == '*':

			args.append(arg)

			arg = step()

		# Just skip whitespaces

		elif curr().isspace():

			step()

		# Something unexpected

		else:

			raise Exception(f"Unexpected symbol '{curr()}' at index #{i} " +

			                f"in the macro declaration:\n'{line}'")

	if arg != '':

		args.append(arg)

	if len(args) > 0:

		print(line, args)

	# Find a comment if any

	comment = ""

	while curr() and curr() != ';': step()

	if curr() == ';':

		step()

		while curr(): comment += step()

	# Find end of the macro

	end_of_macro = False

	while not end_of_macro:

		line = lines[line_idx]

		rbraces = re.finditer('}', line)

		for rbrace_match in rbraces:

			rbrace_idx = rbrace_match.start()

			if line[rbrace_idx - 1] != '\\':

				end_of_macro = True

		line_idx += 1

	# Process comment

	if comment != "":

		comment = comment.lstrip()

		if (len(comment) == 0):

			comment = "!!! EMPTY_COMMENT"

			warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"

		if comment[0].islower():

			warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"

	# Build the output

	line_span = line_idx - line_idx_orig + 1

	result = AsmMacro(line_idx_orig, name, comment, args)

	return (line_span, result)

def get_declarations(asm_file_contents, asm_file_name):

	asm_file_name = asm_file_name.replace("./", "")

	kernel_structure[asm_file_name] = [ [], [], [], [], [] ]

	variable_pattern = re.compile(r'^\s*[\w\.]+\s+d[bwdq]\s+.*')

	macro_pattern = re.compile(r'^\s*macro\s+([\w]+).*')

	proc_pattern = re.compile(r'^\s*proc\s+([\w\.]+).*')

	label_pattern = re.compile(r'^(?!;)\s*([\w\.]+):.*')

	struct_pattern = re.compile(r'^\s*struct\s+([\w]+).*')

	line_idx = 0

	lines = asm_file_contents.splitlines()

	while line_idx < len(lines):

		line = lines[line_idx]

		if variable_pattern.match(line):

			(skip_lines, var) = parse_variable(asm_file_name, lines, line_idx)

			kernel_structure[asm_file_name][VARIABLES].append(var)

			line_idx += skip_lines

			continue

		match = macro_pattern.findall(line)

		if len(match) > 0:

			(skip_lines, macro) = parse_macro(asm_file_name, lines, line_idx)

			kernel_structure[asm_file_name][MACROS].append(macro)

			line_idx += skip_lines

			continue

		match = proc_pattern.findall(line)

		if len(match) > 0:

			proc_name = match[0]

			kernel_structure[asm_file_name][PROCEDURES].append(AsmFunction(line_idx + 1, proc_name))

			line_idx += 1

			continue

		match = label_pattern.findall(line)

		if len(match) > 0:

			label_name = match[0]

			# Don't count local labels

			if label_name[0] != '.':

				kernel_structure[asm_file_name][LABELS].append(AsmLabel(line_idx + 1, label_name))

				line_idx += 1

				continue

		match = struct_pattern.findall(line)

		if len(match) > 0:

			struct_name = match[0]

			kernel_structure[asm_file_name][STRUCTURES].append(AsmStruct(line_idx + 1, struct_name))

			end_of_struct = False

			while not end_of_struct:

				line = lines[line_idx]

				if re.match(r"^ends$", line) != None:

					end_of_struct = True

				line_idx += 1

			continue

		line_idx += 1

def handle_file(handled_files, asm_file_name, subdir = "."):

	if dump_symbols:

		print(f"Handling {asm_file_name}")

	handled_files.append(asm_file_name)

	try:

		asm_file_contents = open(asm_file_name, "r", encoding="utf-8").read()

	except:

		return

	get_declarations(asm_file_contents, asm_file_name)

	include_directive_pattern_1 = re.compile(r'include "(.*)"')

	include_directive_pattern_2 = re.compile(r'include \'(.*)\'')

	includes = include_directive_pattern_1.findall(asm_file_contents)

	includes += include_directive_pattern_2.findall(asm_file_contents)

	for include in includes:

		include = include.replace('\\', '/');

		full_path = subdir + '/' + include;

		if full_path not in handled_files:

			new_subdir = full_path.rsplit('/', 1)[0]

			handle_file(handled_files, full_path, new_subdir)

	return handled_files

kernel_files = []

handle_file(kernel_files, "./kernel.asm");

if dump_symbols:

	for source in kernel_structure:

		print(f"File: {source}")

		if len(kernel_structure[source][VARIABLES]) > 0:

			print(" Variables:")

			for variable in kernel_structure[source][VARIABLES]:

				print(f"  {variable.line}: {variable.name}")

		if len(kernel_structure[source][PROCEDURES]) > 0:

			print(" Procedures:")

			for procedure in kernel_structure[source][PROCEDURES]:

				print(f"  {procedure.line}: {procedure.name}")

		if len(kernel_structure[source][LABELS]) > 0:

			print(" Global labels:")

			for label in kernel_structure[source][LABELS]:

				print(f"  {label.line}: {label.name}")

		if len(kernel_structure[source][MACROS]) > 0:

			print(" Macroses:")

			for macro in kernel_structure[source][MACROS]:

				print(f"  {macro.line}: {macro.name}")

		if len(kernel_structure[source][STRUCTURES]) > 0:

			print(" Structures:")

			for struct in kernel_structure[source][STRUCTURES]:

				print(f"  {struct.line}: {struct.name}")

if print_stats:

	# Collect stats

	var_count = 0

	proc_count = 0

	label_count = 0

	macro_count = 0

	struct_count = 0

	for source in kernel_structure:

		var_count += len(kernel_structure[source][VARIABLES])

		proc_count += len(kernel_structure[source][PROCEDURES])

		label_count += len(kernel_structure[source][LABELS])

		macro_count += len(kernel_structure[source][MACROS])

		struct_count += len(kernel_structure[source][STRUCTURES])

	print(f"File count: {len(kernel_structure)}")

	print(f"Variable count: {var_count}")

	print(f"Procedures count: {proc_count}")

	print(f"Global labels count: {label_count}")

	print(f"Macroses count: {macro_count}")

	print(f"Structures count: {struct_count}")

print(f"Writing doumented sources to {doxygen_src_path}")

created_files = []

def write_something(source, somehing):

	full_path = doxygen_src_path + '/' + source

	# Remove the file on first access if it was created by previous generation

	if full_path not in created_files:

		if os.path.isfile(full_path):

			os.remove(full_path)

		created_files.append(full_path)

	# Only remove the file on 'clean_generated_stuff' flag (removed above, just return)

	if clean_generated_stuff: return

	# Create directories need for the file

	os.makedirs(os.path.dirname(full_path), exist_ok=True)

	f = open(full_path, "a")

	f.write(somehing)

	f.close()

def write_variable(source, variable):

	line = variable.line

	type = variable.type

	init = variable.init

	brief = variable.comment

	name = variable.name.replace(".", "_")

	something = (f"/**\n" +

	             f" * @brief {brief}\n" +

	             f" * @par Initial value\n" +

	             f" * {init}\n" +

	             f" * @par Source\n" +

	             f" * {source}:{line}\n" +

	             f" */\n" +

	             f"{type} {name};\n\n")

	write_something(source, something)

def write_procedure(source, line, name, brief = "Undocumented"):

	name = name.replace(".", "_")

	something = (f"/**\n" +

	             f" * @brief {brief}\n" +

	             f" * @par Source\n" +

	             f" * {source}:{line}\n" +

	             f" */\n" +

	             f"void {name}();\n\n")

	write_something(source, something)

def write_label(source, line, name, brief = "Undocumented"):

	name = name.replace(".", "_")

	something = (f"/**\n" +

	             f" * @brief {brief}\n" +

	             f" * @par Source\n" +

	             f" * {source}:{line}\n" +

	             f" */\n" +

	             f"void {name}();\n\n")

	write_something(source, something)

def write_macro(source, macro):

	if macro.comment == "": brief = "Undocumented"

	else: brief = macro.comment

	line = macro.line

	name = macro.name.replace(".", "_").replace("@", "_")

	# Construct arg list without '['s, ']'s and '*'s

	args = [arg for arg in macro.args if arg not in "[]*"]

	# Construct C-like arg list

	arg_list = ""

	if len(args) > 0:

		arg_list += '('

		argc = 0

		for arg in args:

			if argc != 0:

				arg_list += ", "

			arg_list += arg

			argc += 1

		arg_list += ')'

	something = (f"/**\n" +

	             f" * @def {name}\n" +

	             f" * @brief {brief}\n" +

	             f" * @par Source\n" +

	             f" * {source}:{line}\n" +

	             f" */\n#define {name}{arg_list}\n\n")

	write_something(source, something)

def write_structure(source, line, name, brief = "Undocumented"):

	name = name.replace(".", "_")

	something = (f"/**\n" +

	             f" * @struct {name}\n" +

	             f" * @brief {brief}\n" +

	             f" * @par Source\n" +

	             f" * {source}:{line}\n" +

	             f" */\nstruct {name}" + " {};\n\n")

	write_something(source, something)

i = 1

for source in kernel_structure:

	# Print progress: current/total

	print(f"{i}/{len(kernel_structure)} Writing {source}")

	# Write variables doxygen of the source file

	if len(kernel_structure[source][VARIABLES]) > 0:

		for variable in kernel_structure[source][VARIABLES]:

			write_variable(source, variable)

	if len(kernel_structure[source][PROCEDURES]) > 0:

		for procedure in kernel_structure[source][PROCEDURES]:

			write_procedure(source, procedure.line, procedure.name)

	if len(kernel_structure[source][LABELS]) > 0:

		for label in kernel_structure[source][LABELS]:

			write_label(source, label.line, label.name)

	if len(kernel_structure[source][MACROS]) > 0:

		for macro in kernel_structure[source][MACROS]:

			write_macro(source, macro)

	if len(kernel_structure[source][STRUCTURES]) > 0:

		for structure in kernel_structure[source][STRUCTURES]:

			write_structure(source, structure.line, structure.name)

	i += 1

if enable_warnings:

	open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)