Rev 9923 | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
9249 | Boppan | 1 | #!/usr/bin/python3 |
9312 | Boppan | 2 | # Copyright 2021 Magomed Kostoev |
9249 | Boppan | 3 | # Published under MIT License |
4 | |||
5 | import os |
||
6 | import sys |
||
9314 | Boppan | 7 | import urllib |
9249 | Boppan | 8 | from importlib.machinery import SourceFileLoader |
9310 | Boppan | 9 | from shutil import which |
9249 | Boppan | 10 | import timeit |
11 | import urllib.request |
||
12 | import subprocess |
||
9322 | Boppan | 13 | from threading import Thread |
9340 | Boppan | 14 | import filecmp |
9920 | Boppan | 15 | import traceback |
9922 | Boppan | 16 | import shlex |
9249 | Boppan | 17 | |
18 | sys.path.append('test') |
||
19 | import common |
||
20 | |||
9421 | Boppan | 21 | use_umka = False |
9397 | Boppan | 22 | |
9249 | Boppan | 23 | |
9409 | Boppan | 24 | def log(s, end="\n"): |
25 | print(s, end=end, flush=True) |
||
26 | |||
27 | |||
9922 | Boppan | 28 | def check_retcode(command): |
29 | popen = subprocess.Popen(shlex.split(command)) |
||
30 | return popen.wait() |
||
31 | |||
32 | |||
9409 | Boppan | 33 | def execute(s, mute=False): |
9249 | Boppan | 34 | mute = ">/dev/null" if mute else "" |
35 | code = os.system(f"{s}{mute}") |
||
36 | if code: |
||
37 | print(f"Command returned {code}: \"{s}\"") |
||
38 | exit(-1) |
||
39 | |||
9409 | Boppan | 40 | |
9314 | Boppan | 41 | def download(link, path): |
9409 | Boppan | 42 | log(f"Downloading {path}... ", end="") |
9315 | Boppan | 43 | urllib.request.urlretrieve(link, path) |
9314 | Boppan | 44 | log("Done.") |
45 | |||
9409 | Boppan | 46 | |
9310 | Boppan | 47 | def tool_exists(name): |
48 | assert(type(name) == str) |
||
9409 | Boppan | 49 | return which(name) is not None |
9310 | Boppan | 50 | |
9409 | Boppan | 51 | |
9310 | Boppan | 52 | def check_tools(tools): |
9414 | Boppan | 53 | assert(type(tools) == list) |
9310 | Boppan | 54 | for name_package_pair in tools: |
9414 | Boppan | 55 | assert(type(name_package_pair) == list) |
9310 | Boppan | 56 | assert(len(name_package_pair) == 2) |
57 | assert(type(name_package_pair[0]) == str) |
||
58 | assert(type(name_package_pair[1]) == str) |
||
9409 | Boppan | 59 | |
9310 | Boppan | 60 | not_exists = [] |
61 | for name, package in tools: |
||
62 | if not tool_exists(name): |
||
63 | not_exists.append((name, package)) |
||
64 | if len(not_exists) != 0: |
||
65 | log("Sorry, I can't find some tools:") |
||
9311 | Boppan | 66 | |
67 | header_name = "Name" |
||
68 | header_package = "Package (probably)" |
||
69 | |||
70 | max_name_len = len(header_name) |
||
71 | max_package_name_len = len(header_package) |
||
9310 | Boppan | 72 | for name, package in not_exists: |
73 | if len(package) > max_package_name_len: |
||
74 | max_package_name_len = len(package) |
||
75 | if len(name) > max_name_len: |
||
76 | max_name_len = len(name) |
||
77 | |||
78 | def draw_row(name, package): |
||
9409 | Boppan | 79 | log((f" | {name.ljust(max_name_len)}" + |
80 | f" | {package.ljust(max_package_name_len)} |")) |
||
9310 | Boppan | 81 | |
82 | def draw_line(): |
||
83 | log(f" +-{'-' * max_name_len}-+-{'-' * max_package_name_len}-+") |
||
84 | |||
85 | draw_line() |
||
9311 | Boppan | 86 | draw_row(header_name, header_package) |
9310 | Boppan | 87 | draw_line() |
88 | for name, package in not_exists: |
||
89 | draw_row(name, package) |
||
90 | draw_line() |
||
9931 | boppan | 91 | install_command = 'sudo apt install' |
92 | for _, package in not_exists: |
||
93 | install_command += f' {package}' |
||
94 | log(f"Try to install with:\n {install_command}\n") |
||
9310 | Boppan | 95 | exit() |
96 | |||
9409 | Boppan | 97 | |
9321 | Boppan | 98 | def prepare_test_img(): |
99 | # TODO: Always recompile the kernel (after build system is done?) |
||
9313 | Boppan | 100 | # Get IMG |
101 | if not os.path.exists("kolibri_test.img"): |
||
102 | if len(sys.argv) == 1: |
||
9409 | Boppan | 103 | download("http://builds.kolibrios.org/eng/data/data/kolibri.img", |
104 | "kolibri_test.img") |
||
9313 | Boppan | 105 | else: |
106 | builds_eng = sys.argv[1] |
||
107 | execute(f"cp {builds_eng}/data/data/kolibri.img kolibri_test.img") |
||
9409 | Boppan | 108 | |
9316 | Boppan | 109 | # Open the IMG |
110 | with open("kolibri_test.img", "rb") as img: |
||
111 | img_data = img.read() |
||
112 | img = common.Floppy(img_data) |
||
9321 | Boppan | 113 | |
9316 | Boppan | 114 | # Remove unuseful folders |
115 | img.delete_path("GAMES") |
||
116 | img.delete_path("DEMOS") |
||
117 | img.delete_path("3D") |
||
9409 | Boppan | 118 | |
9313 | Boppan | 119 | # Get test kernel |
120 | if not os.path.exists("kernel.mnt.pretest"): |
||
121 | if len(sys.argv) == 1: |
||
9922 | Boppan | 122 | if check_retcode("tup dbconfig") != 0: |
123 | execute("tup init") |
||
9918 | Boppan | 124 | execute("tup kernel.mnt.pretest") |
9313 | Boppan | 125 | else: |
126 | builds_eng = sys.argv[1] |
||
9409 | Boppan | 127 | kernel_mnt_pretest_subpath = "data/kernel/trunk/kernel.mnt.pretest" |
128 | kernel_mnt_pretest = f"{builds_eng}/{kernel_mnt_pretest_subpath}" |
||
129 | execute(f"cp {kernel_mnt_pretest} kernel.mnt.pretest", mute=True) |
||
130 | |||
9313 | Boppan | 131 | # Put the kernel into IMG |
9316 | Boppan | 132 | with open("kernel.mnt.pretest", "rb") as kernel_mnt_pretest: |
133 | kernel_mnt_pretest_data = kernel_mnt_pretest.read() |
||
9317 | Boppan | 134 | img.add_file_path("KERNEL.MNT", kernel_mnt_pretest_data) |
9316 | Boppan | 135 | img.save("kolibri_test.img") |
9321 | Boppan | 136 | |
9409 | Boppan | 137 | |
9321 | Boppan | 138 | def collect_tests(): |
139 | tests = [] |
||
140 | |||
9313 | Boppan | 141 | # Collect tests from test folder (not recursively yet) |
142 | for test_folder in os.listdir("test"): |
||
143 | test_folder_path = f"test/{test_folder}" |
||
144 | test_file = f"{test_folder_path}/test.py" |
||
9321 | Boppan | 145 | |
9313 | Boppan | 146 | if not os.path.isdir(test_folder_path): |
147 | continue |
||
9321 | Boppan | 148 | |
9313 | Boppan | 149 | if os.path.exists(test_file): |
150 | tests.append(test_folder_path) |
||
9321 | Boppan | 151 | return tests |
152 | |||
9409 | Boppan | 153 | |
9323 | Boppan | 154 | def run_tests_serially_thread(test, root_dir): |
9931 | boppan | 155 | print("\nRunning QEMU tests.") |
9920 | Boppan | 156 | errors = [] |
9313 | Boppan | 157 | test_number = 1 |
158 | for test in tests: |
||
159 | test_dir = f"{root_dir}/{test}" |
||
9409 | Boppan | 160 | |
161 | print(f"[{test_number}/{len(tests)}] {test}... ", end="", flush=True) |
||
9313 | Boppan | 162 | start = timeit.default_timer() |
163 | try: |
||
9409 | Boppan | 164 | loader = SourceFileLoader("test", f"{test_dir}/test.py") |
165 | loader.load_module().run(root_dir, test_dir) |
||
9920 | Boppan | 166 | except common.TestException as exception: |
167 | result = exception.kind() |
||
168 | errors.append((test, exception)) |
||
9313 | Boppan | 169 | else: |
170 | result = "SUCCESS" |
||
171 | finish = timeit.default_timer() |
||
172 | print(f"{result} ({finish - start:.2f} seconds)") |
||
9409 | Boppan | 173 | |
9313 | Boppan | 174 | test_number += 1 |
9920 | Boppan | 175 | if len(errors) != 0: |
176 | print("Some tests failed:") |
||
177 | for error in errors: |
||
178 | test, exception = error |
||
179 | print(f"\n{test}: {str(exception)}\n\nTraceback:") |
||
180 | traceback.print_tb(exception.__traceback__) |
||
181 | print(f"\nQemu command:\n {exception.cmd()}\n") |
||
9322 | Boppan | 182 | |
9923 | Boppan | 183 | |
9323 | Boppan | 184 | def run_tests_serially(tests, root_dir): |
9409 | Boppan | 185 | thread = Thread(target=run_tests_serially_thread, args=(tests, root_dir)) |
9323 | Boppan | 186 | thread.start() |
187 | return thread |
||
188 | |||
9409 | Boppan | 189 | |
9931 | boppan | 190 | def test_umka(): |
191 | class Test: |
||
192 | def __init__(self, path, deps): |
||
193 | self.path = os.path.realpath(path) |
||
194 | self.name = os.path.basename(path) |
||
195 | self.deps = deps |
||
196 | filename_no_ext = os.path.splitext(self.path)[0] |
||
197 | self.ref_log = f"{filename_no_ext}.ref.log" |
||
198 | self.out_log = f"{filename_no_ext}.out.log" |
||
199 | self.ref_png = f"{filename_no_ext}.ref.png" |
||
200 | self.out_png = f"{filename_no_ext}.out.png" |
||
201 | self.log_diff = f"{filename_no_ext}.log.diff" |
||
202 | self.check_png = os.path.exists(self.ref_png) |
||
203 | |||
204 | def find_tests(): |
||
205 | def find_test_dependencies(umka_shell_command_file): |
||
206 | # TODO: Cache the result to not parse tests on each run. |
||
207 | deps = set() |
||
208 | with open(umka_shell_command_file) as f: |
||
209 | test_dir = os.path.dirname(umka_shell_command_file) |
||
210 | for line in f: |
||
211 | parts = line.split() |
||
212 | for dependant in ("disk_add", "ramdisk_init"): |
||
213 | try: |
||
214 | idx = parts.index(dependant) |
||
215 | relative_img_path = parts[idx + 1] |
||
216 | dep_path = f"{test_dir}/{relative_img_path}" |
||
217 | deps.add(os.path.realpath(dep_path)) |
||
218 | except: |
||
219 | pass |
||
220 | return tuple(deps) |
||
221 | |||
222 | tests = [] |
||
223 | for umka_shell_command_file in os.listdir("umka/test"): |
||
224 | umka_shell_command_file = f"umka/test/{umka_shell_command_file}" |
||
225 | if not umka_shell_command_file.endswith(".t"): |
||
226 | continue |
||
227 | if not os.path.isfile(umka_shell_command_file): |
||
228 | continue |
||
229 | deps = find_test_dependencies(umka_shell_command_file) |
||
230 | tests.append(Test(umka_shell_command_file, deps)) |
||
231 | |||
232 | return tests |
||
233 | |||
234 | print("\nCollecting UMKa tests.", flush = True) |
||
235 | tests = find_tests() |
||
236 | # Excluded: #acpi_. |
||
237 | tags_to_tests = ("#xfs_", "#xfsv5_", "#exfat_", "#fat_", "#ext_", "#s05k_", |
||
238 | "#s4k_", "#f30_", "#f70_", "#f70s0_", "#f70s1_", "#f70s5_", |
||
239 | "#lookup_", "#bug_", "#xattr_", "#unicode_", "#draw_", |
||
240 | "#coverage_", "#i40_", "#net_", "#arp_", "#input_", |
||
241 | "#gpt_", "#uevent_") |
||
242 | tests_to_run = [] |
||
243 | for test in tests: |
||
244 | # If none of required tags are in the test name - skip it. |
||
245 | for tag in tags_to_tests: |
||
246 | if tag in test.name: |
||
247 | break |
||
248 | else: |
||
249 | continue |
||
250 | |||
251 | # Check test dependencies. |
||
252 | unmet_deps = [] |
||
253 | for dep in test.deps: |
||
254 | if not os.path.exists(dep): |
||
255 | unmet_deps.append(dep) |
||
256 | |||
257 | if len(unmet_deps) > 0: |
||
258 | print(f"*** WARNING: Test {test.name} has been skipped, unmet dependencies:") |
||
259 | for dep in unmet_deps: |
||
260 | print(f"- {os.path.basename(dep)}") |
||
261 | continue |
||
262 | |||
263 | tests_to_run.append(test) |
||
264 | |||
265 | failed_tests = [] |
||
266 | test_count = len(tests_to_run) |
||
267 | test_i = 1 |
||
268 | print("\nRunning UMKa tests.") |
||
269 | for test in tests_to_run: |
||
270 | print(f"[{test_i}/{test_count}] Running test {test.name}... ", end = "", flush = True) |
||
271 | if os.system(f"(cd umka/test && ../umka_shell -ri {test.path} -o {test.out_log})") != 0: |
||
272 | print("ABORT") |
||
273 | else: |
||
274 | fail_reasons = [] |
||
275 | if not filecmp.cmp(test.out_log, test.ref_log): |
||
276 | fail_reasons.append("log") |
||
277 | if test.check_png and not filecmp.cmp(test.out_png, test.ref_png): |
||
278 | fail_reasons.append("png") |
||
279 | if fail_reasons: |
||
280 | failed_tests.append((test, fail_reasons)) |
||
281 | print("FAILURE") |
||
282 | else: |
||
283 | print("SUCCESS") |
||
284 | test_i += 1 |
||
285 | |||
286 | if len(failed_tests) != 0: |
||
287 | print("\nFailed UMKa tests:") |
||
288 | for failed_test in failed_tests: |
||
289 | test = failed_test[0] |
||
290 | reasons = failed_test[1] |
||
291 | message = f"- {test.name}" |
||
292 | if "log" in reasons: |
||
293 | os.system(f"git --no-pager diff --no-index {test.ref_log} {test.out_log} > {test.log_diff}") |
||
294 | message += f"\n - logs differ: {test.log_diff}" |
||
295 | if "png" in reasons: |
||
296 | message += f"\n - pngs are different:\n" |
||
297 | message += f" - {test.ref_png}\n" |
||
298 | message += f" - {test.out_png}" |
||
299 | print(message) |
||
300 | |||
301 | |||
9923 | Boppan | 302 | def build_umka(): |
9931 | boppan | 303 | print("\nBuilding UMKa... ", end = "", flush = True) |
9397 | Boppan | 304 | env = os.environ |
9931 | boppan | 305 | env["KOLIBRIOS"] = os.path.abspath("../../") |
9923 | Boppan | 306 | env["HOST"] = "linux" |
307 | env["CC"] = "clang" |
||
9931 | boppan | 308 | popen = subprocess.Popen(shlex.split("make --silent -C umka umka_shell default.skn"), env = env) |
9923 | Boppan | 309 | if popen.wait() != 0: |
9931 | boppan | 310 | subprocess.Popen(shlex.split("make --no-print-directory -C umka clean umka_shell default.skn"), env = env) |
311 | if os.system("make --silent -C umka/apps board_cycle") != 0: |
||
312 | os.system("make --no-print-directory -C umka/apps clean board_cycle") |
||
313 | if os.system("make --silent -C umka/tools all") != 0: |
||
314 | os.system("make --no-print-directory -C umka/tools clean all") |
||
315 | print("Done.") |
||
9397 | Boppan | 316 | |
9931 | boppan | 317 | print("\nGenerating images for UMKa tests.", flush = True) |
318 | os.system("(cd umka/img && sudo ./gen.sh)") |
||
9409 | Boppan | 319 | |
9931 | boppan | 320 | |
9414 | Boppan | 321 | def download_umka(): |
322 | if not os.path.exists("umka"): |
||
323 | if os.system("git clone https://github.com/KolibriOS/umka") != 0: |
||
324 | print("Couldn't clone UMKa repo") |
||
325 | exit() |
||
326 | |||
327 | |||
9322 | Boppan | 328 | if __name__ == "__main__": |
329 | root_dir = os.getcwd() |
||
330 | |||
331 | # Check available tools |
||
9923 | Boppan | 332 | tools = [ |
333 | ["qemu-system-i386", "qemu-system-x86"], |
||
334 | ["fasm", "fasm"], |
||
335 | ["tup", "tup"], |
||
336 | ] |
||
9414 | Boppan | 337 | if use_umka: |
9923 | Boppan | 338 | tools.append(["git", "git"]) |
339 | tools.append(["make", "make"]) |
||
9322 | Boppan | 340 | check_tools(tools) |
9409 | Boppan | 341 | |
9322 | Boppan | 342 | prepare_test_img() |
9923 | Boppan | 343 | if use_umka: |
344 | download_umka() |
||
345 | build_umka() |
||
9931 | boppan | 346 | test_umka() |
9322 | Boppan | 347 | tests = collect_tests() |
9323 | Boppan | 348 | serial_executor_thread = run_tests_serially(tests, root_dir) |
9335 | Boppan | 349 | serial_executor_thread.join() |