Subversion Repositories Kolibri OS

Rev

Rev 9340 | Rev 9387 | Go to most recent revision | Only display areas with differences | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 9340 Rev 9342
1
#!/usr/bin/python3
1
#!/usr/bin/python3
2
# Copyright 2021 Magomed Kostoev
2
# Copyright 2021 Magomed Kostoev
3
# Published under MIT License
3
# Published under MIT License
4
 
4
 
5
import os
5
import os
6
import sys
6
import sys
7
import urllib
7
import urllib
8
from importlib.machinery import SourceFileLoader
8
from importlib.machinery import SourceFileLoader
9
from shutil import which
9
from shutil import which
10
import timeit
10
import timeit
11
import urllib.request
11
import urllib.request
12
import subprocess
12
import subprocess
13
from threading import Thread
13
from threading import Thread
14
import filecmp
14
import filecmp
15
 
15
 
16
sys.path.append('test')
16
sys.path.append('test')
17
import common
17
import common
18
 
18
 
19
enable_umka = True
19
enable_umka = True
20
 
20
 
21
def log(s, end = "\n"):
21
def log(s, end = "\n"):
22
    print(s, end = end, flush = True)
22
    print(s, end = end, flush = True)
23
 
23
 
24
def execute(s, mute = False):
24
def execute(s, mute = False):
25
    mute = ">/dev/null" if mute else ""
25
    mute = ">/dev/null" if mute else ""
26
    code = os.system(f"{s}{mute}")
26
    code = os.system(f"{s}{mute}")
27
    if code:
27
    if code:
28
        print(f"Command returned {code}: \"{s}\"")
28
        print(f"Command returned {code}: \"{s}\"")
29
        exit(-1)
29
        exit(-1)
30
 
30
 
31
def stage(name, command, mute = False):
31
def stage(name, command, mute = False):
32
    print(f"{name}... ", end = "")
32
    print(f"{name}... ", end = "")
33
    execute(command, mute = mute)
33
    execute(command, mute = mute)
34
    print("Done.")
34
    print("Done.")
35
 
35
 
36
def download(link, path):
36
def download(link, path):
37
    log(f"Downloading {path}... ", end = "")
37
    log(f"Downloading {path}... ", end = "")
38
    urllib.request.urlretrieve(link, path)
38
    urllib.request.urlretrieve(link, path)
39
    log("Done.")
39
    log("Done.")
40
 
40
 
41
def tool_exists(name):
41
def tool_exists(name):
42
    assert(type(name) == str)
42
    assert(type(name) == str)
43
    return which(name) != None
43
    return which(name) != None
44
 
44
 
45
def check_tools(tools):
45
def check_tools(tools):
46
    assert(type(tools) == tuple)
46
    assert(type(tools) == tuple)
47
    for name_package_pair in tools:
47
    for name_package_pair in tools:
48
        assert(type(name_package_pair) == tuple)
48
        assert(type(name_package_pair) == tuple)
49
        assert(len(name_package_pair) == 2)
49
        assert(len(name_package_pair) == 2)
50
        assert(type(name_package_pair[0]) == str)
50
        assert(type(name_package_pair[0]) == str)
51
        assert(type(name_package_pair[1]) == str)
51
        assert(type(name_package_pair[1]) == str)
52
    
52
    
53
    not_exists = []
53
    not_exists = []
54
    for name, package in tools:
54
    for name, package in tools:
55
        if not tool_exists(name):
55
        if not tool_exists(name):
56
            not_exists.append((name, package))
56
            not_exists.append((name, package))
57
    if len(not_exists) != 0:
57
    if len(not_exists) != 0:
58
        log("Sorry, I can't find some tools:")
58
        log("Sorry, I can't find some tools:")
59
 
59
 
60
        header_name = "Name"
60
        header_name = "Name"
61
        header_package = "Package (probably)"
61
        header_package = "Package (probably)"
62
 
62
 
63
        max_name_len = len(header_name)
63
        max_name_len = len(header_name)
64
        max_package_name_len = len(header_package)
64
        max_package_name_len = len(header_package)
65
        for name, package in not_exists:
65
        for name, package in not_exists:
66
            if len(package) > max_package_name_len:
66
            if len(package) > max_package_name_len:
67
                max_package_name_len = len(package)
67
                max_package_name_len = len(package)
68
            if len(name) > max_name_len:
68
            if len(name) > max_name_len:
69
                max_name_len = len(name)
69
                max_name_len = len(name)
70
 
70
 
71
        def draw_row(name, package):
71
        def draw_row(name, package):
72
            log(f" | {name.ljust(max_name_len)} | {package.ljust(max_package_name_len)} |")
72
            log(f" | {name.ljust(max_name_len)} | {package.ljust(max_package_name_len)} |")
73
 
73
 
74
        def draw_line():
74
        def draw_line():
75
            log(f" +-{'-' * max_name_len}-+-{'-' * max_package_name_len}-+")
75
            log(f" +-{'-' * max_name_len}-+-{'-' * max_package_name_len}-+")
76
 
76
 
77
        draw_line()
77
        draw_line()
78
        draw_row(header_name, header_package)
78
        draw_row(header_name, header_package)
79
        draw_line()
79
        draw_line()
80
        for name, package in not_exists:
80
        for name, package in not_exists:
81
            draw_row(name, package)
81
            draw_row(name, package)
82
        draw_line()
82
        draw_line()
83
        exit()
83
        exit()
84
 
84
 
85
def prepare_test_img():
85
def prepare_test_img():
86
    # TODO: Always recompile the kernel (after build system is done?)
86
    # TODO: Always recompile the kernel (after build system is done?)
87
    # Get IMG
87
    # Get IMG
88
    if not os.path.exists("kolibri_test.img"):
88
    if not os.path.exists("kolibri_test.img"):
89
        if len(sys.argv) == 1:
89
        if len(sys.argv) == 1:
90
            download("http://builds.kolibrios.org/eng/data/data/kolibri.img", "kolibri_test.img")
90
            download("http://builds.kolibrios.org/eng/data/data/kolibri.img", "kolibri_test.img")
91
        else:
91
        else:
92
            builds_eng = sys.argv[1]
92
            builds_eng = sys.argv[1]
93
            execute(f"cp {builds_eng}/data/data/kolibri.img kolibri_test.img")
93
            execute(f"cp {builds_eng}/data/data/kolibri.img kolibri_test.img")
94
    
94
    
95
    # Open the IMG
95
    # Open the IMG
96
    with open("kolibri_test.img", "rb") as img:
96
    with open("kolibri_test.img", "rb") as img:
97
        img_data = img.read()
97
        img_data = img.read()
98
    img = common.Floppy(img_data)
98
    img = common.Floppy(img_data)
99
 
99
 
100
    # Remove unuseful folders
100
    # Remove unuseful folders
101
    img.delete_path("GAMES")
101
    img.delete_path("GAMES")
102
    img.delete_path("DEMOS")
102
    img.delete_path("DEMOS")
103
    img.delete_path("3D")
103
    img.delete_path("3D")
104
    
104
    
105
    # Get test kernel
105
    # Get test kernel
106
    if not os.path.exists("kernel.mnt.pretest"):
106
    if not os.path.exists("kernel.mnt.pretest"):
107
        if len(sys.argv) == 1:
107
        if len(sys.argv) == 1:
108
            with open("lang.inc", "w") as lang_inc:
108
            with open("lang.inc", "w") as lang_inc:
109
                lang_inc.write("lang fix en\n")
109
                lang_inc.write("lang fix en\n")
110
            execute("fasm bootbios.asm bootbios.bin.pretest -dpretest_build=1")
110
            execute("fasm bootbios.asm bootbios.bin.pretest -dpretest_build=1")
111
            execute("fasm -m 65536 kernel.asm kernel.mnt.pretest -dpretest_build=1 -ddebug_com_base=0xe9")
111
            execute("fasm -m 65536 kernel.asm kernel.mnt.pretest -dpretest_build=1 -ddebug_com_base=0xe9")
112
        else:
112
        else:
113
            builds_eng = sys.argv[1]
113
            builds_eng = sys.argv[1]
114
            execute(f"cp {builds_eng}/data/kernel/trunk/kernel.mnt.pretest kernel.mnt.pretest", mute = True)
114
            execute(f"cp {builds_eng}/data/kernel/trunk/kernel.mnt.pretest kernel.mnt.pretest", mute = True)
115
    
115
    
116
    # Put the kernel into IMG
116
    # Put the kernel into IMG
117
    with open("kernel.mnt.pretest", "rb") as kernel_mnt_pretest:
117
    with open("kernel.mnt.pretest", "rb") as kernel_mnt_pretest:
118
        kernel_mnt_pretest_data = kernel_mnt_pretest.read()
118
        kernel_mnt_pretest_data = kernel_mnt_pretest.read()
119
    img.add_file_path("KERNEL.MNT", kernel_mnt_pretest_data)
119
    img.add_file_path("KERNEL.MNT", kernel_mnt_pretest_data)
120
    img.save("kolibri_test.img")
120
    img.save("kolibri_test.img")
121
 
121
 
122
def collect_tests():
122
def collect_tests():
123
    tests = []
123
    tests = []
124
 
124
 
125
    # Collect tests from test folder (not recursively yet)
125
    # Collect tests from test folder (not recursively yet)
126
    for test_folder in os.listdir("test"):
126
    for test_folder in os.listdir("test"):
127
        test_folder_path = f"test/{test_folder}"
127
        test_folder_path = f"test/{test_folder}"
128
        test_file = f"{test_folder_path}/test.py"
128
        test_file = f"{test_folder_path}/test.py"
129
 
129
 
130
        if not os.path.isdir(test_folder_path):
130
        if not os.path.isdir(test_folder_path):
131
            continue
131
            continue
132
 
132
 
133
        if os.path.exists(test_file):
133
        if os.path.exists(test_file):
134
            tests.append(test_folder_path)
134
            tests.append(test_folder_path)
135
    return tests
135
    return tests
136
 
136
 
137
def collect_umka_tests():
137
def collect_umka_tests():
138
    tests = []
138
    tests = []
139
 
139
 
140
    for test_file in os.listdir("umka/test"):
140
    for test_file in os.listdir("umka/test"):
141
        test_file_path = f"umka/test/{test_file}"
141
        test_file_path = f"umka/test/{test_file}"
142
        if not test_file.endswith(".t"):
142
        if not test_file.endswith(".t"):
143
            continue
143
            continue
144
        if not os.path.isfile(test_file_path):
144
        if not os.path.isfile(test_file_path):
145
            continue
145
            continue
146
        tests.append(test_file_path)
146
        tests.append(test_file_path)
147
    return tests
147
    return tests
148
 
148
 
149
def run_tests_serially_thread(test, root_dir):
149
def run_tests_serially_thread(test, root_dir):
150
    test_number = 1
150
    test_number = 1
151
    for test in tests:
151
    for test in tests:
152
        test_dir = f"{root_dir}/{test}"
152
        test_dir = f"{root_dir}/{test}"
153
    
153
    
154
        print(f"[{test_number}/{len(tests)}] {test}... ", end = "", flush=True)
154
        print(f"[{test_number}/{len(tests)}] {test}... ", end = "", flush=True)
155
        start = timeit.default_timer()
155
        start = timeit.default_timer()
156
        try:
156
        try:
157
            SourceFileLoader("test", f"{test_dir}/test.py").load_module().run(root_dir, test_dir)
157
            SourceFileLoader("test", f"{test_dir}/test.py").load_module().run(root_dir, test_dir)
158
        except common.TestTimeoutException:
158
        except common.TestTimeoutException:
159
            result = "TIMEOUT"
159
            result = "TIMEOUT"
160
        except common.TestFailureException:
160
        except common.TestFailureException:
161
            result = "FAILURE"
161
            result = "FAILURE"
162
        else:
162
        else:
163
            result = "SUCCESS"
163
            result = "SUCCESS"
164
        finish = timeit.default_timer()
164
        finish = timeit.default_timer()
165
        print(f"{result} ({finish - start:.2f} seconds)")
165
        print(f"{result} ({finish - start:.2f} seconds)")
166
    
166
    
167
        test_number += 1
167
        test_number += 1
168
 
168
 
169
def run_tests_serially(tests, root_dir):
169
def run_tests_serially(tests, root_dir):
170
    thread = Thread(target = run_tests_serially_thread, args = (tests, root_dir))
170
    thread = Thread(target = run_tests_serially_thread, args = (tests, root_dir))
171
    thread.start()
171
    thread.start()
172
    return thread
172
    return thread
173
 
173
 
174
def gcc(fin, fout):
174
def gcc(fin, fout):
175
    flags = "-m32 -std=c11 -g -O0 -fno-pie -w" # -masm-intel
175
    flags = "-m32 -std=c11 -g -O0 -fno-pie -w"
176
    defines = "-D_FILE_OFFSET_BITS=64 -DNDEBUG -D_POSIX_C_SOURCE=200809L"
176
    defines = "-D_FILE_OFFSET_BITS=64 -DNDEBUG -D_POSIX_C_SOURCE=200809L"
177
    include = "-Iumka -Iumka/linux"
177
    include = "-Iumka -Iumka/linux"
178
    command = f"clang {flags} {defines} {include} -c {fin} -o {fout}"
178
    command = f"clang {flags} {defines} {include} -c {fin} -o {fout}"
179
    print(command)
179
    print(command)
180
    os.system(command)
180
    os.system(command)
181
 
181
 
182
def build_umka_asm():
182
def build_umka_asm():
183
    include = "INCLUDE=\"../../programs/develop/libraries/libcrash/hash\""
183
    include = "INCLUDE=\"../../programs/develop/libraries/libcrash/hash\""
184
    flags = "-dUEFI=1 -dextended_primary_loader=1 -dUMKA=1"
184
    flags = "-dUEFI=1 -dextended_primary_loader=1 -dUMKA=1"
185
    files = "umka/umka.asm umka/build/umka.o -s umka/build/umka.fas"
185
    files = "umka/umka.asm umka/build/umka.o -s umka/build/umka.fas"
186
    memory = "-m 2000000"
186
    memory = "-m 2000000"
187
    command = f"{include} fasm {flags} {files} {memory}"
187
    command = f"{include} fasm {flags} {files} {memory}"
-
 
188
    if sys.platform != "win32":
188
    print(command)
189
        print(command)
189
    os.system(command)
190
        os.system(command)
-
 
191
    else:
-
 
192
        my_env = os.environ.copy()
-
 
193
        my_env["INCLUDE"] = "../../programs/develop/libraries/libcrash/hash"
-
 
194
        print(subprocess.check_output(f"fasm {flags} {files} {memory} -dwin32=1", shell = True, env = my_env))
190
 
195
 
191
def build_umka():
196
def build_umka():
192
    if not enable_umka:
197
    if not enable_umka:
193
        return
198
        return
194
    if os.path.exists("umka_shell.exe"):
199
    if os.path.exists("umka_shell.exe"):
195
        return
200
        return
196
    os.makedirs("umka/build/linux", exist_ok = True)
201
    os.makedirs("umka/build/linux", exist_ok = True)
-
 
202
    os.makedirs("umka/build/win32", exist_ok = True)
197
    sources = [ "umka_shell.c", 
203
    sources = [ "umka_shell.c", 
198
                "shell.c",
204
                "shell.c",
199
                "vdisk.c",
205
                "vdisk.c",
200
                "lodepng.c",
206
                "lodepng.c",
201
                "linux/pci.c",
207
                "getopt.c" ]
-
 
208
    if sys.platform == "win32":
-
 
209
        sources += [ "win32/pci.c", "win32/thread.c" ]
-
 
210
    else:
202
                "linux/thread.c" ]
211
        sources += [ "linux/pci.c", "linux/thread.c" ]
203
    sources = [f"umka/{f}" for f in sources]
212
    sources = [f"umka/{f}" for f in sources]
204
    objects = []
213
    objects = []
205
    for source in sources:
214
    for source in sources:
206
        object_path = source.replace("umka/", "umka/build/")
215
        object_path = source.replace("umka/", "umka/build/")
207
        object_path = f"{object_path}.o"
216
        object_path = f"{object_path}.o"
208
        gcc(source, object_path)
217
        gcc(source, object_path)
209
        objects.append(object_path)
218
        objects.append(object_path)
210
    build_umka_asm()
219
    build_umka_asm()
211
    objects.append("umka/build/umka.o")
220
    objects.append("umka/build/umka.o")
212
    objects = " ".join(objects)
221
    objects = " ".join(objects)
-
 
222
    if sys.platform != "win32":
-
 
223
        ld_script = "-T umka/umka.ld"
-
 
224
    else:
-
 
225
        ld_script = ""
213
    command = f"gcc -m32 -no-pie -o umka_shell.exe -static -T umka/umka.ld {objects}"
226
    command = f"clang -m32 -fno-pie -o umka_shell.exe -static {ld_script} {objects}"
214
    print(command)
227
    print(command)
215
    os.system(command)
228
    os.system(command)
-
 
229
    if not os.path.exists("umka_shell.exe"):
-
 
230
        print("Could't compile umka_shell.exe")
-
 
231
        exit()
216
 
232
 
217
def create_relocated(root_dir, fname):
233
def create_relocated(root_dir, fname):
218
    with open(fname, "rb") as f:
234
    with open(fname, "rb") as f:
219
        contents = f.read()
235
        contents = f.read()
220
    new_contents = contents.replace(b"../img", bytes(f"{root_dir}/umka/img", "ascii"))
236
    new_contents = contents.replace(b"../img", bytes(f"{root_dir}/umka/img", "ascii"))
221
    new_contents = new_contents.replace(b"chess_image.rgb", bytes(f"{root_dir}/umka/test/chess_image.rgb", "ascii"))
237
    new_contents = new_contents.replace(b"chess_image.rgb", bytes(f"{root_dir}/umka/test/chess_image.rgb", "ascii"))
222
    outname = f"{fname}.o" # .o extension just to avoid indexing of the file
238
    outname = f"{fname}.o" # .o extension just to avoid indexing of the file
223
    with open(outname, "wb") as f:
239
    with open(outname, "wb") as f:
224
        f.write(new_contents)
240
        f.write(new_contents)
225
    return outname
241
    return outname
226
 
242
 
227
def run_umka_test(root_dir, test_file_path):
243
def run_umka_test(root_dir, test_file_path):
228
    test = create_relocated(root_dir, test_file_path)
244
    test = create_relocated(root_dir, test_file_path)
229
    ref_log = create_relocated(root_dir, f"{test_file_path[:-2]}.ref.log")
245
    ref_log = create_relocated(root_dir, f"{test_file_path[:-2]}.ref.log")
230
    out_log = f"{test_file_path[:-2]}.out.log.o"
246
    out_log = f"{test_file_path[:-2]}.out.log"
-
 
247
    if sys.platform != "win32":
-
 
248
        prefix = "./"
-
 
249
    else:
-
 
250
        prefix = ""
231
    os.system(f"./umka_shell.exe < {test} > {out_log}")
251
    os.system(f"{prefix}umka_shell.exe < {test} > {out_log}")
-
 
252
    if sys.platform == "win32":
-
 
253
        with open(out_log, "rb") as f:
-
 
254
            contents = f.read()
-
 
255
        contents_no_crlf = contents.replace(b"\r\n", b"\n")
-
 
256
        with open(out_log, "wb") as f:
-
 
257
            f.write(contents_no_crlf)
232
    if not filecmp.cmp(ref_log, out_log):
258
    if not filecmp.cmp(ref_log, out_log):
233
        print(f"FAILURE: {test_file_path}\n", end = "")
259
        print(f"FAILURE: {test_file_path}\n", end = "")
234
    else:
260
    else:
235
        print(f"SUCCESS: {test_file_path}\n", end = "")
261
        print(f"SUCCESS: {test_file_path}\n", end = "")
236
 
262
 
237
if __name__ == "__main__":
263
if __name__ == "__main__":
238
    root_dir = os.getcwd()
264
    root_dir = os.getcwd()
239
 
265
 
240
    # Check available tools
266
    # Check available tools
241
    tools = (("qemu-system-i386", "qemu-system-x86"),
267
    tools = (("qemu-system-i386", "qemu-system-x86"),
242
             ("fasm", "fasm"))
268
             ("fasm", "fasm"))
243
    check_tools(tools)
269
    check_tools(tools)
244
    
270
    
245
    prepare_test_img()
271
    prepare_test_img()
246
    build_umka()
272
    build_umka()
247
    tests = collect_tests()
273
    tests = collect_tests()
248
    umka_tests = collect_umka_tests()
274
    umka_tests = collect_umka_tests()
249
    serial_executor_thread = run_tests_serially(tests, root_dir)
275
    serial_executor_thread = run_tests_serially(tests, root_dir)
250
    if enable_umka:
276
    if enable_umka:
251
        for umka_test in umka_tests:
277
        for umka_test in umka_tests:
252
            run_umka_test(root_dir, umka_test)
278
            run_umka_test(root_dir, umka_test)
253
    serial_executor_thread.join()
279
    serial_executor_thread.join()