Subversion Repositories Kolibri OS

Rev

Rev 8855 | Rev 8957 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
8834 Boppan 1
import re
8825 Boppan 2
import os
8837 Boppan 3
import argparse
8825 Boppan 4
 
8834 Boppan 5
# Parameters
8837 Boppan 6
# Path to doxygen folder to make doxygen files in: -o 
8834 Boppan 7
doxygen_src_path = 'docs/doxygen'
8837 Boppan 8
# Remove generated doxygen files: --clean
9
clean_generated_stuff = False
10
# Dump all defined symbols: --dump
8834 Boppan 11
dump_symbols = False
8837 Boppan 12
# Print symbol stats: --stats
8834 Boppan 13
print_stats = False
8855 Boppan 14
# Do not write warnings file: --nowarn
15
enable_warnings = True
8825 Boppan 16
 
8837 Boppan 17
# Constants
18
link_root = "http://websvn.kolibrios.org/filedetails.php?repname=Kolibri+OS&path=/kernel/trunk"
19
 
8855 Boppan 20
# Warning list
21
warnings = ""
22
 
8837 Boppan 23
# Parse arguments
24
parser = argparse.ArgumentParser()
25
parser.add_argument("-o", help="Doxygen output folder")
26
parser.add_argument("--clean", help="Remove generated files", action="store_true")
27
parser.add_argument("--dump", help="Dump all defined symbols", action="store_true")
28
parser.add_argument("--stats", help="Print symbol stats", action="store_true")
8855 Boppan 29
parser.add_argument("--nowarn", help="Do not write warnings file", action="store_true")
8837 Boppan 30
args = parser.parse_args()
31
doxygen_src_path = args.o if args.o else 'docs/doxygen'
32
clean_generated_stuff = args.clean
33
dump_symbols = args.dump
34
print_stats = args.stats
8855 Boppan 35
enable_warnings = not args.nowarn
8837 Boppan 36
 
8825 Boppan 37
# kernel_structure["filename"] = {
38
#   [ [],    # [0] Variables - [ line, name ]
39
#     [],    # [1] Macros - [ line, name ]
40
#     [],    # [2] Procedures - [ line, name ]
41
#     [],    # [3] Labels - [ line, name ]
42
#     [] ] } # [4] Structures - [ line, name ]
43
VARIABLES = 0
44
MACROS = 1
45
PROCEDURES = 2
46
LABELS = 3
47
STRUCTURES = 4
48
kernel_structure = {}
49
 
8855 Boppan 50
class AsmVariable:
51
	def __init__(self, line, name, type, init, comment, line_span):
52
		self.line = line
53
		self.name = name
54
		self.type = type
55
		self.init = init
56
		self.comment = comment # Comment after the definition (a dd 0 ; Comment)
57
		self.line_span = line_span # How much .asm lines its definition takes
58
 
59
class AsmFunction:
60
	def __init__(self, line, name):
61
		self.line = line
62
		self.name = name
63
 
64
class AsmLabel:
65
	def __init__(self, line, name):
66
		self.line = line
67
		self.name = name
68
 
69
class AsmMacro:
8856 Boppan 70
	def __init__(self, asm_file_name, line, name, comment, args):
71
		self.file = asm_file_name
8855 Boppan 72
		self.line = line
73
		self.name = name
74
		self.comment = comment
75
		self.args = args
76
 
77
class AsmStruct:
78
	def __init__(self, line, name):
79
		self.line = line
80
		self.name = name
81
 
82
def parse_variable(asm_file_name, lines, line_idx):
83
	global warnings
84
 
85
	def curr():
86
		try: return line[i]
87
		except: return ''
88
 
89
	# Returns current and then increments current index
90
	def step():
91
		nonlocal i
92
		c = curr()
93
		i += 1
94
		return c
95
 
96
	line = lines[line_idx]
97
	i = 0
98
	# Skip first spaces
99
	while curr().isspace(): step()
100
	# Get name
101
	name = ""
102
	while curr().isalnum() or curr() == '_' or curr() == '.': name += step()
103
	# Skip spaces after variable name
104
	while curr().isspace(): step()
105
	# Get type specifier (db, dd, etc.)
106
	type = ""
107
	while curr().isalnum() or curr() == '_': type += step()
108
	# Skip spaces after type specifier
109
	while curr().isspace(): step()
110
	# Get initial value (everything up to end of the line or comment)
111
	init = ""
112
	while curr() and curr() != ';': init += step()
113
	# Get comment
114
	comment = ""
115
	if curr() == ';':
116
		step() # Skip ';'
117
		while curr(): comment += step()
118
	# Process type
119
	if type == "db": type = "byte"
120
	elif type == "dw": type = "word"
121
	elif type == "dd": type = "dword"
122
	elif type == "dq": type = "qword"
123
	else: raise Exception(f"Unexpected type: '{type}' (i = {i})")
124
	# Process comment
125
	if comment == "": comment = "Undocumented"
126
	else:
127
		comment = comment.lstrip()
128
		if (len(comment) == 0):
129
			comment = "!!! EMPTY_COMMENT"
130
			warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"
131
		if comment[0].islower():
132
			warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"
133
	# Build the result
134
	result = AsmVariable(line_idx + 1, name, type, init, comment, 1)
135
	return (1, result)
136
 
137
def is_id(c):
138
	return c.isprintable() and c not in "+-/*=<>()[]{}:,|&~#`'\" \n\r\t\v"
139
 
140
def get_comment_begin(line):
141
	result = len(line)
142
	in_str = False
143
	for i in range(len(line)):
144
		if in_str:
145
			if line[i] == in_str: in_str = False
146
			i += 1
147
		elif line[i] == '\'' or line[i] == '\"':
148
			in_str = line[i]
149
			i += 1
150
		elif line[i] == ';':
151
			result = i
152
			break
153
		else:
154
			i += 1
155
	return result
156
 
157
def get_comment(line):
158
	return line[get_comment_begin(line):]
159
 
160
def remove_comment(line):
161
	return line[0:get_comment_begin(line)]
162
 
163
def insert_comment(line, comment):
164
	comment_begin = get_comment_begin(line)
165
	line_left = line[:get_comment_begin(line)]
166
	line_right = line[get_comment_begin(line):]
167
	return line_left + comment + line_right
168
 
169
def has_line_wrap(line):
170
	if remove_comment(line).rstrip()[-1] == '\\':
171
		return True
172
	return False
173
 
174
def remove_line_wrap(line):
175
	if remove_comment(line).rstrip()[-1] == '\\':
176
		return remove_comment(line).rstrip()[:-1]
177
	return line
178
 
179
def parse_macro(asm_file_name, lines, line_idx):
180
	line_idx_orig = line_idx
181
	global warnings
182
 
183
	def curr():
184
		try: return line[i]
185
		except: return ''
186
 
187
	# Returns current and then increments current index
188
	def step():
189
		nonlocal i
190
		c = curr()
191
		i += 1
192
		return c
193
 
194
	line = lines[line_idx]
195
	# Handle line wraps ('\' at the end)
196
	while has_line_wrap(line):
197
		next_line = lines[line_idx + 1]
198
		prev_line_comment = get_comment(line)
199
		line = remove_line_wrap(line) + insert_comment(next_line, prev_line_comment)
200
		line_idx += 1
201
 
202
	i = 0
203
	# Skip first spaces
204
	while curr().isspace(): step()
205
	# Read "macro" keyword
206
	keyword = ""
207
	while is_id(curr()): keyword += step()
208
	if keyword != "macro": raise Exception(f"Not a macro: {line}")
209
	# Skip spaces after "macro"
210
	while curr().isspace(): step()
211
	# Read macro name
212
	name = ""
213
	while curr() and not curr().isspace(): name += step()
214
	# Skip spaces after macro name
215
	while curr().isspace(): step()
216
	# Find all arguments
217
	args = []
218
	arg = ''
219
	while curr() and curr() != ';' and curr() != '{':
220
		# Collect identifier
221
		if is_id(curr()):
222
			arg += step()
223
		# Save the collected identifier
224
		elif curr() == ',':
225
			args.append(arg)
226
			arg = ''
227
			step()
228
		# Just push the '['
229
		elif curr() == '[':
230
			args.append(step())
231
		# Just push the identifier and get ']' ready to be pushed on next comma
232
		elif curr() == ']':
233
			args.append(arg)
234
			arg = step()
235
		# Just push the identifier and get '*' ready to be pushed on next comma
236
		elif curr() == '*':
237
			args.append(arg)
238
			arg = step()
239
		# Just skip whitespaces
240
		elif curr().isspace():
241
			step()
242
		# Something unexpected
243
		else:
244
			raise Exception(f"Unexpected symbol '{curr()}' at index #{i} " +
245
			                f"in the macro declaration:\n'{line}'")
246
	if arg != '':
247
		args.append(arg)
248
	# Find a comment if any
249
	comment = ""
250
	while curr() and curr() != ';': step()
251
	if curr() == ';':
252
		step()
253
		while curr(): comment += step()
254
	# Find end of the macro
255
	end_of_macro = False
256
	while not end_of_macro:
257
		line = lines[line_idx]
258
		rbraces = re.finditer('}', line)
259
		for rbrace_match in rbraces:
260
			rbrace_idx = rbrace_match.start()
261
			if line[rbrace_idx - 1] != '\\':
262
				end_of_macro = True
263
		line_idx += 1
264
	# Process comment
265
	if comment != "":
266
		comment = comment.lstrip()
267
		if (len(comment) == 0):
268
			comment = "!!! EMPTY_COMMENT"
269
			warnings += f"{asm_file_name}:{line_idx + 1}: Empty comment in\n"
270
		if comment[0].islower():
271
			warnings += f"{asm_file_name}:{line_idx + 1}: Сomment sarting with lowercase\n"
272
	# Build the output
273
	line_span = line_idx - line_idx_orig + 1
8856 Boppan 274
	result = AsmMacro(asm_file_name, line_idx_orig, name, comment, args)
8855 Boppan 275
	return (line_span, result)
276
 
8825 Boppan 277
def get_declarations(asm_file_contents, asm_file_name):
8842 Boppan 278
	asm_file_name = asm_file_name.replace("./", "")
8825 Boppan 279
	kernel_structure[asm_file_name] = [ [], [], [], [], [] ]
280
 
8855 Boppan 281
	variable_pattern = re.compile(r'^\s*[\w\.]+\s+d[bwdq]\s+.*')
8825 Boppan 282
	macro_pattern = re.compile(r'^\s*macro\s+([\w]+).*')
283
	proc_pattern = re.compile(r'^\s*proc\s+([\w\.]+).*')
284
	label_pattern = re.compile(r'^(?!;)\s*([\w\.]+):.*')
285
	struct_pattern = re.compile(r'^\s*struct\s+([\w]+).*')
286
 
287
	line_idx = 0
288
	lines = asm_file_contents.splitlines()
289
	while line_idx < len(lines):
290
		line = lines[line_idx]
291
 
8855 Boppan 292
		if variable_pattern.match(line):
293
			(skip_lines, var) = parse_variable(asm_file_name, lines, line_idx)
294
			kernel_structure[asm_file_name][VARIABLES].append(var)
295
			line_idx += skip_lines
8825 Boppan 296
			continue
297
 
298
		match = macro_pattern.findall(line)
299
		if len(match) > 0:
8855 Boppan 300
			(skip_lines, macro) = parse_macro(asm_file_name, lines, line_idx)
301
			kernel_structure[asm_file_name][MACROS].append(macro)
302
			line_idx += skip_lines
8825 Boppan 303
			continue
304
 
305
		match = proc_pattern.findall(line)
306
		if len(match) > 0:
307
			proc_name = match[0]
8855 Boppan 308
			kernel_structure[asm_file_name][PROCEDURES].append(AsmFunction(line_idx + 1, proc_name))
8825 Boppan 309
			line_idx += 1
310
			continue
311
 
312
		match = label_pattern.findall(line)
313
		if len(match) > 0:
314
			label_name = match[0]
315
			# Don't count local labels
316
			if label_name[0] != '.':
8855 Boppan 317
				kernel_structure[asm_file_name][LABELS].append(AsmLabel(line_idx + 1, label_name))
8825 Boppan 318
				line_idx += 1
319
				continue
320
 
321
		match = struct_pattern.findall(line)
322
		if len(match) > 0:
323
			struct_name = match[0]
8855 Boppan 324
			kernel_structure[asm_file_name][STRUCTURES].append(AsmStruct(line_idx + 1, struct_name))
8825 Boppan 325
			end_of_struct = False
326
			while not end_of_struct:
327
				line = lines[line_idx]
328
				if re.match(r"^ends$", line) != None:
329
					end_of_struct = True
330
				line_idx += 1
331
			continue
332
 
333
		line_idx += 1
334
 
8834 Boppan 335
def handle_file(handled_files, asm_file_name, subdir = "."):
8841 Boppan 336
	if dump_symbols:
337
		print(f"Handling {asm_file_name}")
8825 Boppan 338
	handled_files.append(asm_file_name)
339
	try:
340
		asm_file_contents = open(asm_file_name, "r", encoding="utf-8").read()
341
	except:
342
		return
343
	get_declarations(asm_file_contents, asm_file_name)
344
	include_directive_pattern_1 = re.compile(r'include "(.*)"')
345
	include_directive_pattern_2 = re.compile(r'include \'(.*)\'')
346
	includes = include_directive_pattern_1.findall(asm_file_contents)
347
	includes += include_directive_pattern_2.findall(asm_file_contents)
348
	for include in includes:
349
		include = include.replace('\\', '/');
350
		full_path = subdir + '/' + include;
351
		if full_path not in handled_files:
352
			new_subdir = full_path.rsplit('/', 1)[0]
8834 Boppan 353
			handle_file(handled_files, full_path, new_subdir)
8825 Boppan 354
	return handled_files
355
 
356
kernel_files = []
357
 
8834 Boppan 358
handle_file(kernel_files, "./kernel.asm");
8825 Boppan 359
 
8834 Boppan 360
if dump_symbols:
361
	for source in kernel_structure:
362
		print(f"File: {source}")
363
		if len(kernel_structure[source][VARIABLES]) > 0:
364
			print(" Variables:")
365
			for variable in kernel_structure[source][VARIABLES]:
8855 Boppan 366
				print(f"  {variable.line}: {variable.name}")
8834 Boppan 367
		if len(kernel_structure[source][PROCEDURES]) > 0:
368
			print(" Procedures:")
369
			for procedure in kernel_structure[source][PROCEDURES]:
8855 Boppan 370
				print(f"  {procedure.line}: {procedure.name}")
8834 Boppan 371
		if len(kernel_structure[source][LABELS]) > 0:
372
			print(" Global labels:")
373
			for label in kernel_structure[source][LABELS]:
8855 Boppan 374
				print(f"  {label.line}: {label.name}")
8834 Boppan 375
		if len(kernel_structure[source][MACROS]) > 0:
376
			print(" Macroses:")
377
			for macro in kernel_structure[source][MACROS]:
8855 Boppan 378
				print(f"  {macro.line}: {macro.name}")
8834 Boppan 379
		if len(kernel_structure[source][STRUCTURES]) > 0:
380
			print(" Structures:")
381
			for struct in kernel_structure[source][STRUCTURES]:
8855 Boppan 382
				print(f"  {struct.line}: {struct.name}")
8825 Boppan 383
 
8834 Boppan 384
if print_stats:
385
	# Collect stats
386
	var_count = 0
387
	proc_count = 0
388
	label_count = 0
389
	macro_count = 0
390
	struct_count = 0
8825 Boppan 391
 
8834 Boppan 392
	for source in kernel_structure:
393
		var_count += len(kernel_structure[source][VARIABLES])
394
		proc_count += len(kernel_structure[source][PROCEDURES])
395
		label_count += len(kernel_structure[source][LABELS])
396
		macro_count += len(kernel_structure[source][MACROS])
397
		struct_count += len(kernel_structure[source][STRUCTURES])
8825 Boppan 398
 
8841 Boppan 399
	print(f"File count: {len(kernel_structure)}")
8834 Boppan 400
	print(f"Variable count: {var_count}")
401
	print(f"Procedures count: {proc_count}")
402
	print(f"Global labels count: {label_count}")
403
	print(f"Macroses count: {macro_count}")
404
	print(f"Structures count: {struct_count}")
8825 Boppan 405
 
8834 Boppan 406
print(f"Writing doumented sources to {doxygen_src_path}")
407
 
408
created_files = []
409
 
8842 Boppan 410
def write_something(source, somehing):
8834 Boppan 411
	full_path = doxygen_src_path + '/' + source
412
	# Remove the file on first access if it was created by previous generation
413
	if full_path not in created_files:
414
		if os.path.isfile(full_path):
415
			os.remove(full_path)
416
		created_files.append(full_path)
417
	# Only remove the file on 'clean_generated_stuff' flag (removed above, just return)
418
	if clean_generated_stuff: return
419
	# Create directories need for the file
420
	os.makedirs(os.path.dirname(full_path), exist_ok=True)
421
	f = open(full_path, "a")
8842 Boppan 422
	f.write(somehing)
8834 Boppan 423
	f.close()
424
 
8855 Boppan 425
def write_variable(source, variable):
426
	line = variable.line
427
	type = variable.type
428
	init = variable.init
429
	brief = variable.comment
430
	name = variable.name.replace(".", "_")
8842 Boppan 431
	something = (f"/**\n" +
432
	             f" * @brief {brief}\n" +
433
	             f" * @par Initial value\n" +
434
	             f" * {init}\n" +
435
	             f" * @par Source\n" +
436
	             f" * {source}:{line}\n" +
437
	             f" */\n" +
438
	             f"{type} {name};\n\n")
439
	write_something(source, something)
440
 
441
def write_procedure(source, line, name, brief = "Undocumented"):
442
	name = name.replace(".", "_")
443
	something = (f"/**\n" +
444
	             f" * @brief {brief}\n" +
445
	             f" * @par Source\n" +
446
	             f" * {source}:{line}\n" +
447
	             f" */\n" +
448
	             f"void {name}();\n\n")
449
	write_something(source, something)
450
 
8844 Boppan 451
def write_label(source, line, name, brief = "Undocumented"):
452
	name = name.replace(".", "_")
453
	something = (f"/**\n" +
454
	             f" * @brief {brief}\n" +
455
	             f" * @par Source\n" +
456
	             f" * {source}:{line}\n" +
457
	             f" */\n" +
458
	             f"void {name}();\n\n")
459
	write_something(source, something)
460
 
8855 Boppan 461
def write_macro(source, macro):
462
	if macro.comment == "": brief = "Undocumented"
463
	else: brief = macro.comment
464
	line = macro.line
465
	name = macro.name.replace(".", "_").replace("@", "_")
466
	# Construct arg list without '['s, ']'s and '*'s
467
	args = [arg for arg in macro.args if arg not in "[]*"]
468
	# Construct C-like arg list
469
	arg_list = ""
470
	if len(args) > 0:
471
		arg_list += '('
472
		argc = 0
473
		for arg in args:
474
			if argc != 0:
475
				arg_list += ", "
476
			arg_list += arg
477
			argc += 1
478
		arg_list += ')'
479
 
8846 Boppan 480
	something = (f"/**\n" +
481
	             f" * @def {name}\n" +
482
	             f" * @brief {brief}\n" +
483
	             f" * @par Source\n" +
484
	             f" * {source}:{line}\n" +
8855 Boppan 485
	             f" */\n#define {name}{arg_list}\n\n")
8846 Boppan 486
	write_something(source, something)
487
 
488
def write_structure(source, line, name, brief = "Undocumented"):
489
	name = name.replace(".", "_")
490
	something = (f"/**\n" +
491
	             f" * @struct {name}\n" +
492
	             f" * @brief {brief}\n" +
493
	             f" * @par Source\n" +
494
	             f" * {source}:{line}\n" +
495
	             f" */\nstruct {name}" + " {};\n\n")
496
	write_something(source, something)
497
 
8834 Boppan 498
i = 1
499
for source in kernel_structure:
500
	# Print progress: current/total
501
	print(f"{i}/{len(kernel_structure)} Writing {source}")
502
	# Write variables doxygen of the source file
503
	if len(kernel_structure[source][VARIABLES]) > 0:
504
		for variable in kernel_structure[source][VARIABLES]:
8855 Boppan 505
			write_variable(source, variable)
8842 Boppan 506
	if len(kernel_structure[source][PROCEDURES]) > 0:
507
		for procedure in kernel_structure[source][PROCEDURES]:
8855 Boppan 508
			write_procedure(source, procedure.line, procedure.name)
8844 Boppan 509
	if len(kernel_structure[source][LABELS]) > 0:
510
		for label in kernel_structure[source][LABELS]:
8855 Boppan 511
			write_label(source, label.line, label.name)
8846 Boppan 512
	if len(kernel_structure[source][MACROS]) > 0:
513
		for macro in kernel_structure[source][MACROS]:
8855 Boppan 514
			write_macro(source, macro)
8846 Boppan 515
	if len(kernel_structure[source][STRUCTURES]) > 0:
516
		for structure in kernel_structure[source][STRUCTURES]:
8855 Boppan 517
			write_structure(source, structure.line, structure.name)
8834 Boppan 518
	i += 1
8855 Boppan 519
 
520
if enable_warnings:
521
	open('asmxygen.txt', "w", encoding = "utf-8").write(warnings)