Rev 9420 | Rev 9918 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
9249 | Boppan | 1 | #!/usr/bin/python3 |
9312 | Boppan | 2 | # Copyright 2021 Magomed Kostoev |
9249 | Boppan | 3 | # Published under MIT License |
4 | |||
5 | import os |
||
6 | import sys |
||
9314 | Boppan | 7 | import urllib |
9249 | Boppan | 8 | from importlib.machinery import SourceFileLoader |
9310 | Boppan | 9 | from shutil import which |
9249 | Boppan | 10 | import timeit |
11 | import urllib.request |
||
12 | import subprocess |
||
9322 | Boppan | 13 | from threading import Thread |
9340 | Boppan | 14 | import filecmp |
9249 | Boppan | 15 | |
16 | sys.path.append('test') |
||
17 | import common |
||
18 | |||
9421 | Boppan | 19 | use_umka = False |
9397 | Boppan | 20 | |
9249 | Boppan | 21 | |
9409 | Boppan | 22 | def log(s, end="\n"): |
23 | print(s, end=end, flush=True) |
||
24 | |||
25 | |||
26 | def execute(s, mute=False): |
||
9249 | Boppan | 27 | mute = ">/dev/null" if mute else "" |
28 | code = os.system(f"{s}{mute}") |
||
29 | if code: |
||
30 | print(f"Command returned {code}: \"{s}\"") |
||
31 | exit(-1) |
||
32 | |||
9409 | Boppan | 33 | |
34 | def stage(name, command, mute=False): |
||
35 | print(f"{name}... ", end="") |
||
36 | execute(command, mute=mute) |
||
9249 | Boppan | 37 | print("Done.") |
38 | |||
9409 | Boppan | 39 | |
9314 | Boppan | 40 | def download(link, path): |
9409 | Boppan | 41 | log(f"Downloading {path}... ", end="") |
9315 | Boppan | 42 | urllib.request.urlretrieve(link, path) |
9314 | Boppan | 43 | log("Done.") |
44 | |||
9409 | Boppan | 45 | |
9310 | Boppan | 46 | def tool_exists(name): |
47 | assert(type(name) == str) |
||
9409 | Boppan | 48 | return which(name) is not None |
9310 | Boppan | 49 | |
9409 | Boppan | 50 | |
9310 | Boppan | 51 | def check_tools(tools): |
9414 | Boppan | 52 | assert(type(tools) == list) |
9310 | Boppan | 53 | for name_package_pair in tools: |
9414 | Boppan | 54 | assert(type(name_package_pair) == list) |
9310 | Boppan | 55 | assert(len(name_package_pair) == 2) |
56 | assert(type(name_package_pair[0]) == str) |
||
57 | assert(type(name_package_pair[1]) == str) |
||
9409 | Boppan | 58 | |
9310 | Boppan | 59 | not_exists = [] |
60 | for name, package in tools: |
||
61 | if not tool_exists(name): |
||
62 | not_exists.append((name, package)) |
||
63 | if len(not_exists) != 0: |
||
64 | log("Sorry, I can't find some tools:") |
||
9311 | Boppan | 65 | |
66 | header_name = "Name" |
||
67 | header_package = "Package (probably)" |
||
68 | |||
69 | max_name_len = len(header_name) |
||
70 | max_package_name_len = len(header_package) |
||
9310 | Boppan | 71 | for name, package in not_exists: |
72 | if len(package) > max_package_name_len: |
||
73 | max_package_name_len = len(package) |
||
74 | if len(name) > max_name_len: |
||
75 | max_name_len = len(name) |
||
76 | |||
77 | def draw_row(name, package): |
||
9409 | Boppan | 78 | log((f" | {name.ljust(max_name_len)}" + |
79 | f" | {package.ljust(max_package_name_len)} |")) |
||
9310 | Boppan | 80 | |
81 | def draw_line(): |
||
82 | log(f" +-{'-' * max_name_len}-+-{'-' * max_package_name_len}-+") |
||
83 | |||
84 | draw_line() |
||
9311 | Boppan | 85 | draw_row(header_name, header_package) |
9310 | Boppan | 86 | draw_line() |
87 | for name, package in not_exists: |
||
88 | draw_row(name, package) |
||
89 | draw_line() |
||
90 | exit() |
||
91 | |||
9409 | Boppan | 92 | |
9321 | Boppan | 93 | def prepare_test_img(): |
94 | # TODO: Always recompile the kernel (after build system is done?) |
||
9313 | Boppan | 95 | # Get IMG |
96 | if not os.path.exists("kolibri_test.img"): |
||
97 | if len(sys.argv) == 1: |
||
9409 | Boppan | 98 | download("http://builds.kolibrios.org/eng/data/data/kolibri.img", |
99 | "kolibri_test.img") |
||
9313 | Boppan | 100 | else: |
101 | builds_eng = sys.argv[1] |
||
102 | execute(f"cp {builds_eng}/data/data/kolibri.img kolibri_test.img") |
||
9409 | Boppan | 103 | |
9316 | Boppan | 104 | # Open the IMG |
105 | with open("kolibri_test.img", "rb") as img: |
||
106 | img_data = img.read() |
||
107 | img = common.Floppy(img_data) |
||
9321 | Boppan | 108 | |
9316 | Boppan | 109 | # Remove unuseful folders |
110 | img.delete_path("GAMES") |
||
111 | img.delete_path("DEMOS") |
||
112 | img.delete_path("3D") |
||
9409 | Boppan | 113 | |
9313 | Boppan | 114 | # Get test kernel |
115 | if not os.path.exists("kernel.mnt.pretest"): |
||
116 | if len(sys.argv) == 1: |
||
117 | with open("lang.inc", "w") as lang_inc: |
||
118 | lang_inc.write("lang fix en\n") |
||
119 | execute("fasm bootbios.asm bootbios.bin.pretest -dpretest_build=1") |
||
9409 | Boppan | 120 | command = "fasm " |
121 | command += "-dpretest_build=1 -ddebug_com_base=0xe9 " |
||
122 | command += "-m 65536 " |
||
123 | command += "kernel.asm kernel.mnt.pretest" |
||
124 | execute(command) |
||
9313 | Boppan | 125 | else: |
126 | builds_eng = sys.argv[1] |
||
9409 | Boppan | 127 | kernel_mnt_pretest_subpath = "data/kernel/trunk/kernel.mnt.pretest" |
128 | kernel_mnt_pretest = f"{builds_eng}/{kernel_mnt_pretest_subpath}" |
||
129 | execute(f"cp {kernel_mnt_pretest} kernel.mnt.pretest", mute=True) |
||
130 | |||
9313 | Boppan | 131 | # Put the kernel into IMG |
9316 | Boppan | 132 | with open("kernel.mnt.pretest", "rb") as kernel_mnt_pretest: |
133 | kernel_mnt_pretest_data = kernel_mnt_pretest.read() |
||
9317 | Boppan | 134 | img.add_file_path("KERNEL.MNT", kernel_mnt_pretest_data) |
9316 | Boppan | 135 | img.save("kolibri_test.img") |
9321 | Boppan | 136 | |
9409 | Boppan | 137 | |
9321 | Boppan | 138 | def collect_tests(): |
139 | tests = [] |
||
140 | |||
9313 | Boppan | 141 | # Collect tests from test folder (not recursively yet) |
142 | for test_folder in os.listdir("test"): |
||
143 | test_folder_path = f"test/{test_folder}" |
||
144 | test_file = f"{test_folder_path}/test.py" |
||
9321 | Boppan | 145 | |
9313 | Boppan | 146 | if not os.path.isdir(test_folder_path): |
147 | continue |
||
9321 | Boppan | 148 | |
9313 | Boppan | 149 | if os.path.exists(test_file): |
150 | tests.append(test_folder_path) |
||
9321 | Boppan | 151 | return tests |
152 | |||
9409 | Boppan | 153 | |
9323 | Boppan | 154 | def run_tests_serially_thread(test, root_dir): |
9313 | Boppan | 155 | test_number = 1 |
156 | for test in tests: |
||
157 | test_dir = f"{root_dir}/{test}" |
||
9409 | Boppan | 158 | |
159 | print(f"[{test_number}/{len(tests)}] {test}... ", end="", flush=True) |
||
9313 | Boppan | 160 | start = timeit.default_timer() |
161 | try: |
||
9409 | Boppan | 162 | loader = SourceFileLoader("test", f"{test_dir}/test.py") |
163 | loader.load_module().run(root_dir, test_dir) |
||
9313 | Boppan | 164 | except common.TestTimeoutException: |
165 | result = "TIMEOUT" |
||
166 | except common.TestFailureException: |
||
167 | result = "FAILURE" |
||
168 | else: |
||
169 | result = "SUCCESS" |
||
170 | finish = timeit.default_timer() |
||
171 | print(f"{result} ({finish - start:.2f} seconds)") |
||
9409 | Boppan | 172 | |
9313 | Boppan | 173 | test_number += 1 |
9322 | Boppan | 174 | |
9409 | Boppan | 175 | |
9323 | Boppan | 176 | def run_tests_serially(tests, root_dir): |
9409 | Boppan | 177 | thread = Thread(target=run_tests_serially_thread, args=(tests, root_dir)) |
9323 | Boppan | 178 | thread.start() |
179 | return thread |
||
180 | |||
9409 | Boppan | 181 | |
9397 | Boppan | 182 | def build_umka_asm(object_output_dir): |
183 | umka_o = f"{object_output_dir}/umka.o" |
||
9409 | Boppan | 184 | kolibri_kernel_trunk_runtests_py = os.path.abspath(__file__) |
185 | kolibri_kernel_trunk = os.path.dirname(kolibri_kernel_trunk_runtests_py) |
||
186 | kolibri_kernel = os.path.dirname(kolibri_kernel_trunk) |
||
187 | kolibrios_folder = os.path.dirname(kolibri_kernel) |
||
9397 | Boppan | 188 | env = os.environ |
9409 | Boppan | 189 | libcrash = "programs/develop/libraries/libcrash/hash" |
9397 | Boppan | 190 | env["INCLUDE"] = "" |
191 | env["INCLUDE"] += f"{kolibrios_folder}/kernel/trunk;" |
||
9409 | Boppan | 192 | env["INCLUDE"] += f"{kolibrios_folder}/{libcrash}" |
193 | command = "fasm " |
||
9412 | Boppan | 194 | command += "-dWIN32=1 " if sys.platform == "win32" else "" |
9409 | Boppan | 195 | command += "-dUEFI=1 -dextended_primary_loader=1 -dUMKA=1 " |
196 | command += "umka/umka.asm umka/build/umka.o -s umka/build/umka.fas " |
||
197 | command += "-m 2000000 " |
||
9412 | Boppan | 198 | print(command) |
9409 | Boppan | 199 | stdout = subprocess.check_output(command, shell=True, env=env) |
9412 | Boppan | 200 | print(stdout.decode("ascii")) |
9397 | Boppan | 201 | return umka_o |
202 | |||
9409 | Boppan | 203 | |
9397 | Boppan | 204 | def cc(src, obj, include_path): |
9418 | Boppan | 205 | if tool_exists("i686-w64-mingw32-gcc"): |
206 | compiler = "i686-w64-mingw32-gcc" |
||
207 | elif tool_exists("clang"): |
||
208 | compiler = "clang" |
||
209 | else: |
||
210 | print("No compiler found to compile UMKa (tried i686-w64-mingw32-gcc and clang)") |
||
211 | print(" Please make sure you have installed llvm-mingw or clang") |
||
212 | print(" If they're installled consider updating PATH variable") |
||
213 | command = f"{compiler} " |
||
9409 | Boppan | 214 | command += "-Wno-everything -std=c11 -g -O0 -fno-pie -m32 -masm=intel -c " |
215 | command += "-D_FILE_OFFSET_BITS=64 -DNDEBUG -D_POSIX_C_SOURCE=200809L " |
||
9397 | Boppan | 216 | command += f"-I {include_path} {src} -o {obj}" |
9412 | Boppan | 217 | print(command) |
9397 | Boppan | 218 | if os.system(command) != 0: |
219 | exit() |
||
220 | |||
9409 | Boppan | 221 | |
9397 | Boppan | 222 | def link(objects): |
9418 | Boppan | 223 | if tool_exists("i686-w64-mingw32-gcc"): |
224 | compiler = "i686-w64-mingw32-gcc" |
||
225 | elif tool_exists("clang"): |
||
226 | compiler = "clang" |
||
227 | else: |
||
228 | print("No compiler found to compile UMKa (tried i686-w64-mingw32-gcc and clang)") |
||
229 | print(" Please make sure you have installed llvm-mingw or clang") |
||
230 | print(" If they're installled consider updating PATH variable") |
||
9412 | Boppan | 231 | if sys.platform == "linux" or sys.platform == "linux2": |
232 | linker_script = "-T umka/umka.ld" |
||
233 | else: |
||
234 | linker_script = "-Wl,/ALIGN:65536 -Wl,/MAP:umka.map " |
||
9418 | Boppan | 235 | command = f"{compiler} " |
9409 | Boppan | 236 | command += "-Wno-everything " |
9412 | Boppan | 237 | command += f"-no-pie -m32 -o umka_shell.exe -static {linker_script} " |
9397 | Boppan | 238 | command += " ".join(objects) |
9412 | Boppan | 239 | print(command) |
9397 | Boppan | 240 | if os.system(command) != 0: |
241 | exit() |
||
242 | |||
9409 | Boppan | 243 | |
9397 | Boppan | 244 | def build_umka(): |
245 | if not use_umka: |
||
246 | return |
||
247 | |||
9409 | Boppan | 248 | os.makedirs("umka/build", exist_ok=True) |
9397 | Boppan | 249 | |
9412 | Boppan | 250 | platform = "win32" if sys.platform == "win32" else "linux" |
251 | |||
252 | os.makedirs(f"umka/build/{platform}", exist_ok=True) |
||
253 | |||
9397 | Boppan | 254 | c_sources = [ |
255 | "umka_shell.c", |
||
256 | "shell.c", |
||
257 | "trace.c", |
||
258 | "trace_lbr.c", |
||
259 | "vdisk.c", |
||
260 | "vnet.c", |
||
9412 | Boppan | 261 | "getopt.c", |
262 | "isatty.c", |
||
9397 | Boppan | 263 | "lodepng.c", |
9412 | Boppan | 264 | f"{platform}/pci.c", |
265 | f"{platform}/thread.c", |
||
9397 | Boppan | 266 | "util.c", |
267 | ] |
||
268 | |||
9409 | Boppan | 269 | src_obj_pairs = [ |
9412 | Boppan | 270 | (f"umka/{source}", f"umka/build/{source}.o") for source in c_sources |
9409 | Boppan | 271 | ] |
9397 | Boppan | 272 | |
273 | for src, obj in src_obj_pairs: |
||
274 | cc(src, obj, "umka/linux") |
||
275 | |||
276 | umka_o = build_umka_asm("umka/build") |
||
277 | |||
9409 | Boppan | 278 | objects = [obj for src, obj in src_obj_pairs] + [umka_o] |
9397 | Boppan | 279 | link(objects) |
280 | |||
281 | os.chdir("umka/test") |
||
9409 | Boppan | 282 | for test in [t for t in os.listdir(".") if t.endswith(".t")]: |
9397 | Boppan | 283 | out_log = f"{test[:-2]}.out.log" |
284 | ref_log = f"{test[:-2]}.ref.log" |
||
9412 | Boppan | 285 | cmd_umka = f"..{os.sep}..{os.sep}umka_shell.exe < {test} > {out_log}" |
9397 | Boppan | 286 | print(cmd_umka) |
287 | os.system(cmd_umka) |
||
9412 | Boppan | 288 | with open(out_log, "rb") as f: |
289 | crlf = bytes([0x0D, 0x0A]) |
||
290 | lf = bytes([0x0A]) |
||
291 | out_log_contents = f.read().replace(crlf, lf) |
||
292 | with open(out_log, "wb") as f: |
||
293 | f.write(out_log_contents) |
||
294 | with open(ref_log, "rb") as f: |
||
295 | ref_log_contents = f.read() |
||
296 | if out_log_contents != ref_log_contents: |
||
9397 | Boppan | 297 | print("FAILURE") |
298 | exit() |
||
299 | os.chdir("../../") |
||
300 | print("SUCCESS") |
||
301 | |||
9414 | Boppan | 302 | |
303 | def download_umka(): |
||
9415 | Boppan | 304 | if not use_umka: |
305 | return |
||
306 | |||
9414 | Boppan | 307 | if not os.path.exists("umka"): |
308 | if os.system("git clone https://github.com/KolibriOS/umka") != 0: |
||
309 | print("Couldn't clone UMKa repo") |
||
310 | exit() |
||
311 | os.chdir("umka") |
||
9420 | Boppan | 312 | if os.system("git checkout trunk") != 0: |
313 | print("Couldn't checkout trunk branch of UMKa") |
||
314 | exit() |
||
9414 | Boppan | 315 | os.system("git pull") |
316 | os.chdir("../") |
||
317 | |||
318 | |||
319 | def download_umka_imgs(): |
||
9415 | Boppan | 320 | if not use_umka: |
321 | return |
||
322 | |||
9414 | Boppan | 323 | imgs = [ |
324 | "fat32_test0.img", |
||
325 | "jfs.img", |
||
326 | "kolibri.img", |
||
327 | "xfs_borg_bit.img", |
||
328 | "xfs_v4_btrees_l2.img", |
||
329 | "xfs_v4_files_s05k_b4k_n8k.img", |
||
330 | "xfs_v4_ftype0_s05k_b2k_n8k_xattr.img", |
||
331 | "xfs_v4_ftype0_s05k_b2k_n8k.img", |
||
332 | "xfs_v4_ftype0_s4k_b4k_n8k.img", |
||
333 | "xfs_v4_ftype1_s05k_b2k_n8k.img", |
||
334 | "xfs_v4_unicode.img", |
||
335 | "xfs_v4_xattr.img", |
||
336 | "xfs_v5_files_s05k_b4k_n8k.img", |
||
337 | "xfs_v5_ftype1_s05k_b2k_n8k.img", |
||
338 | ] |
||
339 | |||
340 | for img in imgs: |
||
9416 | Boppan | 341 | if not os.path.exists(f"umka/img/{img}"): |
342 | download(f"http://ftp.kolibrios.org/users/Boppan/img/{img}", |
||
343 | f"umka/img/{img}") |
||
9414 | Boppan | 344 | |
9322 | Boppan | 345 | if __name__ == "__main__": |
346 | root_dir = os.getcwd() |
||
347 | |||
348 | # Check available tools |
||
9414 | Boppan | 349 | tools = [["qemu-system-i386", "qemu-system-x86"], |
350 | ["fasm", "fasm"]] |
||
351 | if use_umka: |
||
352 | tools.append(["git", "git"]); |
||
9322 | Boppan | 353 | check_tools(tools) |
9409 | Boppan | 354 | |
9322 | Boppan | 355 | prepare_test_img() |
9414 | Boppan | 356 | download_umka() |
357 | download_umka_imgs() |
||
9397 | Boppan | 358 | build_umka() |
9322 | Boppan | 359 | tests = collect_tests() |
9323 | Boppan | 360 | serial_executor_thread = run_tests_serially(tests, root_dir) |
9335 | Boppan | 361 | serial_executor_thread.join()> |