Rev 9340 | Rev 9387 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 9340 | Rev 9342 | ||
---|---|---|---|
1 | #!/usr/bin/python3 |
1 | #!/usr/bin/python3 |
2 | # Copyright 2021 Magomed Kostoev |
2 | # Copyright 2021 Magomed Kostoev |
3 | # Published under MIT License |
3 | # Published under MIT License |
4 | 4 | ||
5 | import os |
5 | import os |
6 | import sys |
6 | import sys |
7 | import urllib |
7 | import urllib |
8 | from importlib.machinery import SourceFileLoader |
8 | from importlib.machinery import SourceFileLoader |
9 | from shutil import which |
9 | from shutil import which |
10 | import timeit |
10 | import timeit |
11 | import urllib.request |
11 | import urllib.request |
12 | import subprocess |
12 | import subprocess |
13 | from threading import Thread |
13 | from threading import Thread |
14 | import filecmp |
14 | import filecmp |
15 | 15 | ||
16 | sys.path.append('test') |
16 | sys.path.append('test') |
17 | import common |
17 | import common |
18 | 18 | ||
19 | enable_umka = True |
19 | enable_umka = True |
20 | 20 | ||
21 | def log(s, end = "\n"): |
21 | def log(s, end = "\n"): |
22 | print(s, end = end, flush = True) |
22 | print(s, end = end, flush = True) |
23 | 23 | ||
24 | def execute(s, mute = False): |
24 | def execute(s, mute = False): |
25 | mute = ">/dev/null" if mute else "" |
25 | mute = ">/dev/null" if mute else "" |
26 | code = os.system(f"{s}{mute}") |
26 | code = os.system(f"{s}{mute}") |
27 | if code: |
27 | if code: |
28 | print(f"Command returned {code}: \"{s}\"") |
28 | print(f"Command returned {code}: \"{s}\"") |
29 | exit(-1) |
29 | exit(-1) |
30 | 30 | ||
31 | def stage(name, command, mute = False): |
31 | def stage(name, command, mute = False): |
32 | print(f"{name}... ", end = "") |
32 | print(f"{name}... ", end = "") |
33 | execute(command, mute = mute) |
33 | execute(command, mute = mute) |
34 | print("Done.") |
34 | print("Done.") |
35 | 35 | ||
36 | def download(link, path): |
36 | def download(link, path): |
37 | log(f"Downloading {path}... ", end = "") |
37 | log(f"Downloading {path}... ", end = "") |
38 | urllib.request.urlretrieve(link, path) |
38 | urllib.request.urlretrieve(link, path) |
39 | log("Done.") |
39 | log("Done.") |
40 | 40 | ||
41 | def tool_exists(name): |
41 | def tool_exists(name): |
42 | assert(type(name) == str) |
42 | assert(type(name) == str) |
43 | return which(name) != None |
43 | return which(name) != None |
44 | 44 | ||
45 | def check_tools(tools): |
45 | def check_tools(tools): |
46 | assert(type(tools) == tuple) |
46 | assert(type(tools) == tuple) |
47 | for name_package_pair in tools: |
47 | for name_package_pair in tools: |
48 | assert(type(name_package_pair) == tuple) |
48 | assert(type(name_package_pair) == tuple) |
49 | assert(len(name_package_pair) == 2) |
49 | assert(len(name_package_pair) == 2) |
50 | assert(type(name_package_pair[0]) == str) |
50 | assert(type(name_package_pair[0]) == str) |
51 | assert(type(name_package_pair[1]) == str) |
51 | assert(type(name_package_pair[1]) == str) |
52 | 52 | ||
53 | not_exists = [] |
53 | not_exists = [] |
54 | for name, package in tools: |
54 | for name, package in tools: |
55 | if not tool_exists(name): |
55 | if not tool_exists(name): |
56 | not_exists.append((name, package)) |
56 | not_exists.append((name, package)) |
57 | if len(not_exists) != 0: |
57 | if len(not_exists) != 0: |
58 | log("Sorry, I can't find some tools:") |
58 | log("Sorry, I can't find some tools:") |
59 | 59 | ||
60 | header_name = "Name" |
60 | header_name = "Name" |
61 | header_package = "Package (probably)" |
61 | header_package = "Package (probably)" |
62 | 62 | ||
63 | max_name_len = len(header_name) |
63 | max_name_len = len(header_name) |
64 | max_package_name_len = len(header_package) |
64 | max_package_name_len = len(header_package) |
65 | for name, package in not_exists: |
65 | for name, package in not_exists: |
66 | if len(package) > max_package_name_len: |
66 | if len(package) > max_package_name_len: |
67 | max_package_name_len = len(package) |
67 | max_package_name_len = len(package) |
68 | if len(name) > max_name_len: |
68 | if len(name) > max_name_len: |
69 | max_name_len = len(name) |
69 | max_name_len = len(name) |
70 | 70 | ||
71 | def draw_row(name, package): |
71 | def draw_row(name, package): |
72 | log(f" | {name.ljust(max_name_len)} | {package.ljust(max_package_name_len)} |") |
72 | log(f" | {name.ljust(max_name_len)} | {package.ljust(max_package_name_len)} |") |
73 | 73 | ||
74 | def draw_line(): |
74 | def draw_line(): |
75 | log(f" +-{'-' * max_name_len}-+-{'-' * max_package_name_len}-+") |
75 | log(f" +-{'-' * max_name_len}-+-{'-' * max_package_name_len}-+") |
76 | 76 | ||
77 | draw_line() |
77 | draw_line() |
78 | draw_row(header_name, header_package) |
78 | draw_row(header_name, header_package) |
79 | draw_line() |
79 | draw_line() |
80 | for name, package in not_exists: |
80 | for name, package in not_exists: |
81 | draw_row(name, package) |
81 | draw_row(name, package) |
82 | draw_line() |
82 | draw_line() |
83 | exit() |
83 | exit() |
84 | 84 | ||
85 | def prepare_test_img(): |
85 | def prepare_test_img(): |
86 | # TODO: Always recompile the kernel (after build system is done?) |
86 | # TODO: Always recompile the kernel (after build system is done?) |
87 | # Get IMG |
87 | # Get IMG |
88 | if not os.path.exists("kolibri_test.img"): |
88 | if not os.path.exists("kolibri_test.img"): |
89 | if len(sys.argv) == 1: |
89 | if len(sys.argv) == 1: |
90 | download("http://builds.kolibrios.org/eng/data/data/kolibri.img", "kolibri_test.img") |
90 | download("http://builds.kolibrios.org/eng/data/data/kolibri.img", "kolibri_test.img") |
91 | else: |
91 | else: |
92 | builds_eng = sys.argv[1] |
92 | builds_eng = sys.argv[1] |
93 | execute(f"cp {builds_eng}/data/data/kolibri.img kolibri_test.img") |
93 | execute(f"cp {builds_eng}/data/data/kolibri.img kolibri_test.img") |
94 | 94 | ||
95 | # Open the IMG |
95 | # Open the IMG |
96 | with open("kolibri_test.img", "rb") as img: |
96 | with open("kolibri_test.img", "rb") as img: |
97 | img_data = img.read() |
97 | img_data = img.read() |
98 | img = common.Floppy(img_data) |
98 | img = common.Floppy(img_data) |
99 | 99 | ||
100 | # Remove unuseful folders |
100 | # Remove unuseful folders |
101 | img.delete_path("GAMES") |
101 | img.delete_path("GAMES") |
102 | img.delete_path("DEMOS") |
102 | img.delete_path("DEMOS") |
103 | img.delete_path("3D") |
103 | img.delete_path("3D") |
104 | 104 | ||
105 | # Get test kernel |
105 | # Get test kernel |
106 | if not os.path.exists("kernel.mnt.pretest"): |
106 | if not os.path.exists("kernel.mnt.pretest"): |
107 | if len(sys.argv) == 1: |
107 | if len(sys.argv) == 1: |
108 | with open("lang.inc", "w") as lang_inc: |
108 | with open("lang.inc", "w") as lang_inc: |
109 | lang_inc.write("lang fix en\n") |
109 | lang_inc.write("lang fix en\n") |
110 | execute("fasm bootbios.asm bootbios.bin.pretest -dpretest_build=1") |
110 | execute("fasm bootbios.asm bootbios.bin.pretest -dpretest_build=1") |
111 | execute("fasm -m 65536 kernel.asm kernel.mnt.pretest -dpretest_build=1 -ddebug_com_base=0xe9") |
111 | execute("fasm -m 65536 kernel.asm kernel.mnt.pretest -dpretest_build=1 -ddebug_com_base=0xe9") |
112 | else: |
112 | else: |
113 | builds_eng = sys.argv[1] |
113 | builds_eng = sys.argv[1] |
114 | execute(f"cp {builds_eng}/data/kernel/trunk/kernel.mnt.pretest kernel.mnt.pretest", mute = True) |
114 | execute(f"cp {builds_eng}/data/kernel/trunk/kernel.mnt.pretest kernel.mnt.pretest", mute = True) |
115 | 115 | ||
116 | # Put the kernel into IMG |
116 | # Put the kernel into IMG |
117 | with open("kernel.mnt.pretest", "rb") as kernel_mnt_pretest: |
117 | with open("kernel.mnt.pretest", "rb") as kernel_mnt_pretest: |
118 | kernel_mnt_pretest_data = kernel_mnt_pretest.read() |
118 | kernel_mnt_pretest_data = kernel_mnt_pretest.read() |
119 | img.add_file_path("KERNEL.MNT", kernel_mnt_pretest_data) |
119 | img.add_file_path("KERNEL.MNT", kernel_mnt_pretest_data) |
120 | img.save("kolibri_test.img") |
120 | img.save("kolibri_test.img") |
121 | 121 | ||
122 | def collect_tests(): |
122 | def collect_tests(): |
123 | tests = [] |
123 | tests = [] |
124 | 124 | ||
125 | # Collect tests from test folder (not recursively yet) |
125 | # Collect tests from test folder (not recursively yet) |
126 | for test_folder in os.listdir("test"): |
126 | for test_folder in os.listdir("test"): |
127 | test_folder_path = f"test/{test_folder}" |
127 | test_folder_path = f"test/{test_folder}" |
128 | test_file = f"{test_folder_path}/test.py" |
128 | test_file = f"{test_folder_path}/test.py" |
129 | 129 | ||
130 | if not os.path.isdir(test_folder_path): |
130 | if not os.path.isdir(test_folder_path): |
131 | continue |
131 | continue |
132 | 132 | ||
133 | if os.path.exists(test_file): |
133 | if os.path.exists(test_file): |
134 | tests.append(test_folder_path) |
134 | tests.append(test_folder_path) |
135 | return tests |
135 | return tests |
136 | 136 | ||
137 | def collect_umka_tests(): |
137 | def collect_umka_tests(): |
138 | tests = [] |
138 | tests = [] |
139 | 139 | ||
140 | for test_file in os.listdir("umka/test"): |
140 | for test_file in os.listdir("umka/test"): |
141 | test_file_path = f"umka/test/{test_file}" |
141 | test_file_path = f"umka/test/{test_file}" |
142 | if not test_file.endswith(".t"): |
142 | if not test_file.endswith(".t"): |
143 | continue |
143 | continue |
144 | if not os.path.isfile(test_file_path): |
144 | if not os.path.isfile(test_file_path): |
145 | continue |
145 | continue |
146 | tests.append(test_file_path) |
146 | tests.append(test_file_path) |
147 | return tests |
147 | return tests |
148 | 148 | ||
149 | def run_tests_serially_thread(test, root_dir): |
149 | def run_tests_serially_thread(test, root_dir): |
150 | test_number = 1 |
150 | test_number = 1 |
151 | for test in tests: |
151 | for test in tests: |
152 | test_dir = f"{root_dir}/{test}" |
152 | test_dir = f"{root_dir}/{test}" |
153 | 153 | ||
154 | print(f"[{test_number}/{len(tests)}] {test}... ", end = "", flush=True) |
154 | print(f"[{test_number}/{len(tests)}] {test}... ", end = "", flush=True) |
155 | start = timeit.default_timer() |
155 | start = timeit.default_timer() |
156 | try: |
156 | try: |
157 | SourceFileLoader("test", f"{test_dir}/test.py").load_module().run(root_dir, test_dir) |
157 | SourceFileLoader("test", f"{test_dir}/test.py").load_module().run(root_dir, test_dir) |
158 | except common.TestTimeoutException: |
158 | except common.TestTimeoutException: |
159 | result = "TIMEOUT" |
159 | result = "TIMEOUT" |
160 | except common.TestFailureException: |
160 | except common.TestFailureException: |
161 | result = "FAILURE" |
161 | result = "FAILURE" |
162 | else: |
162 | else: |
163 | result = "SUCCESS" |
163 | result = "SUCCESS" |
164 | finish = timeit.default_timer() |
164 | finish = timeit.default_timer() |
165 | print(f"{result} ({finish - start:.2f} seconds)") |
165 | print(f"{result} ({finish - start:.2f} seconds)") |
166 | 166 | ||
167 | test_number += 1 |
167 | test_number += 1 |
168 | 168 | ||
169 | def run_tests_serially(tests, root_dir): |
169 | def run_tests_serially(tests, root_dir): |
170 | thread = Thread(target = run_tests_serially_thread, args = (tests, root_dir)) |
170 | thread = Thread(target = run_tests_serially_thread, args = (tests, root_dir)) |
171 | thread.start() |
171 | thread.start() |
172 | return thread |
172 | return thread |
173 | 173 | ||
174 | def gcc(fin, fout): |
174 | def gcc(fin, fout): |
175 | flags = "-m32 -std=c11 -g -O0 -fno-pie -w" # -masm-intel |
175 | flags = "-m32 -std=c11 -g -O0 -fno-pie -w" |
176 | defines = "-D_FILE_OFFSET_BITS=64 -DNDEBUG -D_POSIX_C_SOURCE=200809L" |
176 | defines = "-D_FILE_OFFSET_BITS=64 -DNDEBUG -D_POSIX_C_SOURCE=200809L" |
177 | include = "-Iumka -Iumka/linux" |
177 | include = "-Iumka -Iumka/linux" |
178 | command = f"clang {flags} {defines} {include} -c {fin} -o {fout}" |
178 | command = f"clang {flags} {defines} {include} -c {fin} -o {fout}" |
179 | print(command) |
179 | print(command) |
180 | os.system(command) |
180 | os.system(command) |
181 | 181 | ||
182 | def build_umka_asm(): |
182 | def build_umka_asm(): |
183 | include = "INCLUDE=\"../../programs/develop/libraries/libcrash/hash\"" |
183 | include = "INCLUDE=\"../../programs/develop/libraries/libcrash/hash\"" |
184 | flags = "-dUEFI=1 -dextended_primary_loader=1 -dUMKA=1" |
184 | flags = "-dUEFI=1 -dextended_primary_loader=1 -dUMKA=1" |
185 | files = "umka/umka.asm umka/build/umka.o -s umka/build/umka.fas" |
185 | files = "umka/umka.asm umka/build/umka.o -s umka/build/umka.fas" |
186 | memory = "-m 2000000" |
186 | memory = "-m 2000000" |
187 | command = f"{include} fasm {flags} {files} {memory}" |
187 | command = f"{include} fasm {flags} {files} {memory}" |
- | 188 | if sys.platform != "win32": |
|
188 | print(command) |
189 | print(command) |
189 | os.system(command) |
190 | os.system(command) |
- | 191 | else: |
|
- | 192 | my_env = os.environ.copy() |
|
- | 193 | my_env["INCLUDE"] = "../../programs/develop/libraries/libcrash/hash" |
|
- | 194 | print(subprocess.check_output(f"fasm {flags} {files} {memory} -dwin32=1", shell = True, env = my_env)) |
|
190 | 195 | ||
191 | def build_umka(): |
196 | def build_umka(): |
192 | if not enable_umka: |
197 | if not enable_umka: |
193 | return |
198 | return |
194 | if os.path.exists("umka_shell.exe"): |
199 | if os.path.exists("umka_shell.exe"): |
195 | return |
200 | return |
196 | os.makedirs("umka/build/linux", exist_ok = True) |
201 | os.makedirs("umka/build/linux", exist_ok = True) |
- | 202 | os.makedirs("umka/build/win32", exist_ok = True) |
|
197 | sources = [ "umka_shell.c", |
203 | sources = [ "umka_shell.c", |
198 | "shell.c", |
204 | "shell.c", |
199 | "vdisk.c", |
205 | "vdisk.c", |
200 | "lodepng.c", |
206 | "lodepng.c", |
201 | "linux/pci.c", |
207 | "getopt.c" ] |
- | 208 | if sys.platform == "win32": |
|
- | 209 | sources += [ "win32/pci.c", "win32/thread.c" ] |
|
- | 210 | else: |
|
202 | "linux/thread.c" ] |
211 | sources += [ "linux/pci.c", "linux/thread.c" ] |
203 | sources = [f"umka/{f}" for f in sources] |
212 | sources = [f"umka/{f}" for f in sources] |
204 | objects = [] |
213 | objects = [] |
205 | for source in sources: |
214 | for source in sources: |
206 | object_path = source.replace("umka/", "umka/build/") |
215 | object_path = source.replace("umka/", "umka/build/") |
207 | object_path = f"{object_path}.o" |
216 | object_path = f"{object_path}.o" |
208 | gcc(source, object_path) |
217 | gcc(source, object_path) |
209 | objects.append(object_path) |
218 | objects.append(object_path) |
210 | build_umka_asm() |
219 | build_umka_asm() |
211 | objects.append("umka/build/umka.o") |
220 | objects.append("umka/build/umka.o") |
212 | objects = " ".join(objects) |
221 | objects = " ".join(objects) |
- | 222 | if sys.platform != "win32": |
|
- | 223 | ld_script = "-T umka/umka.ld" |
|
- | 224 | else: |
|
- | 225 | ld_script = "" |
|
213 | command = f"gcc -m32 -no-pie -o umka_shell.exe -static -T umka/umka.ld {objects}" |
226 | command = f"clang -m32 -fno-pie -o umka_shell.exe -static {ld_script} {objects}" |
214 | print(command) |
227 | print(command) |
215 | os.system(command) |
228 | os.system(command) |
- | 229 | if not os.path.exists("umka_shell.exe"): |
|
- | 230 | print("Could't compile umka_shell.exe") |
|
- | 231 | exit() |
|
216 | 232 | ||
217 | def create_relocated(root_dir, fname): |
233 | def create_relocated(root_dir, fname): |
218 | with open(fname, "rb") as f: |
234 | with open(fname, "rb") as f: |
219 | contents = f.read() |
235 | contents = f.read() |
220 | new_contents = contents.replace(b"../img", bytes(f"{root_dir}/umka/img", "ascii")) |
236 | new_contents = contents.replace(b"../img", bytes(f"{root_dir}/umka/img", "ascii")) |
221 | new_contents = new_contents.replace(b"chess_image.rgb", bytes(f"{root_dir}/umka/test/chess_image.rgb", "ascii")) |
237 | new_contents = new_contents.replace(b"chess_image.rgb", bytes(f"{root_dir}/umka/test/chess_image.rgb", "ascii")) |
222 | outname = f"{fname}.o" # .o extension just to avoid indexing of the file |
238 | outname = f"{fname}.o" # .o extension just to avoid indexing of the file |
223 | with open(outname, "wb") as f: |
239 | with open(outname, "wb") as f: |
224 | f.write(new_contents) |
240 | f.write(new_contents) |
225 | return outname |
241 | return outname |
226 | 242 | ||
227 | def run_umka_test(root_dir, test_file_path): |
243 | def run_umka_test(root_dir, test_file_path): |
228 | test = create_relocated(root_dir, test_file_path) |
244 | test = create_relocated(root_dir, test_file_path) |
229 | ref_log = create_relocated(root_dir, f"{test_file_path[:-2]}.ref.log") |
245 | ref_log = create_relocated(root_dir, f"{test_file_path[:-2]}.ref.log") |
230 | out_log = f"{test_file_path[:-2]}.out.log.o" |
246 | out_log = f"{test_file_path[:-2]}.out.log" |
- | 247 | if sys.platform != "win32": |
|
- | 248 | prefix = "./" |
|
- | 249 | else: |
|
- | 250 | prefix = "" |
|
231 | os.system(f"./umka_shell.exe < {test} > {out_log}") |
251 | os.system(f"{prefix}umka_shell.exe < {test} > {out_log}") |
- | 252 | if sys.platform == "win32": |
|
- | 253 | with open(out_log, "rb") as f: |
|
- | 254 | contents = f.read() |
|
- | 255 | contents_no_crlf = contents.replace(b"\r\n", b"\n") |
|
- | 256 | with open(out_log, "wb") as f: |
|
- | 257 | f.write(contents_no_crlf) |
|
232 | if not filecmp.cmp(ref_log, out_log): |
258 | if not filecmp.cmp(ref_log, out_log): |
233 | print(f"FAILURE: {test_file_path}\n", end = "") |
259 | print(f"FAILURE: {test_file_path}\n", end = "") |
234 | else: |
260 | else: |
235 | print(f"SUCCESS: {test_file_path}\n", end = "") |
261 | print(f"SUCCESS: {test_file_path}\n", end = "") |
236 | 262 | ||
237 | if __name__ == "__main__": |
263 | if __name__ == "__main__": |
238 | root_dir = os.getcwd() |
264 | root_dir = os.getcwd() |
239 | 265 | ||
240 | # Check available tools |
266 | # Check available tools |
241 | tools = (("qemu-system-i386", "qemu-system-x86"), |
267 | tools = (("qemu-system-i386", "qemu-system-x86"), |
242 | ("fasm", "fasm")) |
268 | ("fasm", "fasm")) |
243 | check_tools(tools) |
269 | check_tools(tools) |
244 | 270 | ||
245 | prepare_test_img() |
271 | prepare_test_img() |
246 | build_umka() |
272 | build_umka() |
247 | tests = collect_tests() |
273 | tests = collect_tests() |
248 | umka_tests = collect_umka_tests() |
274 | umka_tests = collect_umka_tests() |
249 | serial_executor_thread = run_tests_serially(tests, root_dir) |
275 | serial_executor_thread = run_tests_serially(tests, root_dir) |
250 | if enable_umka: |
276 | if enable_umka: |
251 | for umka_test in umka_tests: |
277 | for umka_test in umka_tests: |
252 | run_umka_test(root_dir, umka_test) |
278 | run_umka_test(root_dir, umka_test) |
253 | serial_executor_thread.join()> |
279 | serial_executor_thread.join()> |