| 0,0 → 1,292 |
|---|
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| ;; ;; |
| ;; Copyright (C) KolibriOS team 2004-2013. All rights reserved. ;; |
| ;; Distributed under terms of the GNU General Public License ;; |
| ;; ;; |
| ;; Written by CleverMouse ;; |
| ;; ;; |
| ;; GNU GENERAL PUBLIC LICENSE ;; |
| ;; Version 2, June 1991 ;; |
| ;; ;; |
| ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; |
| |
| uglobal |
| |
| utf8_bytes_rest dd ? ; bytes rest in current UTF8 sequence |
| utf8_char dd ? ; first bits of current UTF8 character |
| |
| endg |
| |
| |
| ;get_next_byte: |
| ;; Load next byte from the packet, translating to cp866 if necessary |
| ;; At input esi = pointer to data, edx = limit of data |
| ;; Output is either (translated) byte in al with CF set or CF cleared. |
| ; mov eax, [encoding] |
| ; jmp [get_byte_table+eax*4] |
| ; |
| ;get_byte_cp866: |
| ; cmp esi, edx |
| ; jae .nothing |
| ; lodsb |
| ;.nothing: |
| ; ret |
| ; |
| ;get_byte_cp1251: |
| ; cmp esi, edx |
| ; jae .nothing |
| ; lodsb |
| ; cmp al, 0x80 |
| ; jb @f |
| ; and eax, 0x7F |
| ; mov al, [cp1251_table+eax] |
| ;@@: |
| ; stc |
| ;.nothing: |
| ; ret |
| |
| get_byte_utf8: |
| ; UTF8 decoding is slightly complicated. |
| ; One character can occupy one or more bytes. |
| ; The boundary in packets theoretically can be anywhere in data, |
| ; so this procedure keeps internal state between calls and handles |
| ; one byte at a time, looping until character is read or packet is over. |
| ; Globally, there are two distinct tasks: decode byte sequence to unicode char |
| ; and convert this unicode char to our base encoding (that is cp866). |
| ; 1. Check that there are data. |
| cmp esi, edx |
| jae .nothing |
| ; 2. Load byte. |
| lodsb |
| movzx ecx, al |
| ; 3. Bytes in an UTF8 sequence can be of any of three types. |
| ; If most significant bit is cleared, sequence is one byte and usual ASCII char. |
| ; First byte of a sequence must be 11xxxxxx, other bytes are 10yyyyyy. |
| and al, 0xC0 |
| jns .single_byte |
| jp .first_byte |
| ; 4. This byte is not first in UTF8 sequence. |
| ; 4a. Check that the sequence was started. If no, it is invalid byte |
| ; and we simply ignore it. |
| cmp [utf8_bytes_rest], 0 |
| jz get_byte_utf8 |
| ; 4b. Otherwise, it is really next byte and it gives some more bits of char. |
| mov eax, [utf8_char] |
| shl eax, 6 |
| lea eax, [eax+ecx-0x80] |
| ; 4c. Decrement number of bytes rest in the sequence. |
| ; If it goes to zero, character is read, so return it. |
| dec [utf8_bytes_rest] |
| jz .got_char |
| mov [utf8_char], eax |
| jmp get_byte_utf8 |
| ; 5. If the byte is first in UTF8 sequence, calculate the number of leading 1s |
| ; - it equals total number of bytes in the sequence; some other bits rest for |
| ; leading bits in the character. |
| .first_byte: |
| mov eax, -1 |
| @@: |
| inc eax |
| add cl, cl |
| js @b |
| mov [utf8_bytes_rest], eax |
| xchg eax, ecx |
| inc ecx |
| shr al, cl |
| mov [utf8_char], eax |
| jmp get_byte_utf8 |
| ; 6. If the byte is ASCII char, it is the character. |
| .single_byte: |
| xchg eax, ecx |
| .got_char: |
| ; We got the character, now abandon a possible sequence in progress. |
| and [utf8_bytes_rest], 0 |
| ; Now second task. The unicode character is in eax, and now we shall convert it |
| ; to cp866. |
| cmp eax, 0x80 |
| jb .done |
| ; 0x410-0x43F -> 0x80-0xAF, 0x440-0x44F -> 0xE0-0xEF, 0x401 -> 0xF0, 0x451 -> 0xF1 |
| cmp eax, 0x401 |
| jz .YO |
| cmp eax, 0x451 |
| jz .yo |
| cmp eax, 0x410 |
| jb .unrecognized |
| cmp eax, 0x440 |
| jb .part1 |
| cmp eax, 0x450 |
| jb .part2 |
| cmp eax, 0x25a0 |
| jae .unrecognized |
| sub eax, 0x2500 |
| jb .unrecognized |
| mov al, [cp866_boxes+eax] |
| ret |
| .part1: |
| sub al, 0x10-0x80 |
| .nothing: |
| .done: |
| ret |
| .part2: |
| sub al, (0x40-0xE0) and 0xFF |
| ret |
| .unrecognized: |
| mov al, '?' |
| stc |
| ret |
| .YO: |
| mov al, 0xF0 |
| stc |
| ret |
| .yo: |
| mov al, 0xF1 |
| stc |
| ret |
| |
| |
| |
| ;recode_to_cp866: |
| ; rep movsb |
| ; ret |
| ; |
| ;recode_to_cp1251: |
| ; xor eax, eax |
| ; jecxz .nothing |
| ; .loop: |
| ; lodsb |
| ; cmp al,0x80 |
| ; jb @f |
| ; mov al, [cp866_table-0x80+eax] |
| ; @@: stosb |
| ; loop .loop |
| ; .nothing: |
| ; ret |
| |
| recode_to_utf8: |
| jecxz .nothing |
| .loop: |
| lodsb |
| cmp al, 0x80 |
| jb .single_byte |
| and eax, 0x7F |
| mov ax, [utf8_table+eax*2] |
| stosw |
| loop .loop |
| ret |
| .single_byte: |
| stosb |
| loop .loop |
| .nothing: |
| ret |
| |
| ;recode: |
| ; mov eax, [encoding] |
| ; jmp [recode_proc+eax*4] |
| |
| |
| |
| ;encoding dd UTF8 |
| ;recode_proc dd recode_to_cp866, recode_to_cp1251, recode_to_utf8 |
| ;get_byte_table dd get_byte_cp866, get_byte_cp1251, get_byte_utf8 |
| |
| |
| ;cp1251_table: |
| ; db '?','?','?','?','?','?','?','?' , '?','?','?','?','?','?','?','?' ; 8 |
| ; db '?','?','?','?','?',$F9,'?','?' , '?','?','?','?','?','?','?','?' ; 9 |
| ; db '?',$F6,$F7,'?',$FD,'?','?','?' , $F0,'?',$F2,'?','?','?','?',$F4 ; A |
| ; db $F8,'?','?','?','?','?','?',$FA , $F1,$FC,$F3,'?','?','?','?',$F5 ; B |
| ; db $80,$81,$82,$83,$84,$85,$86,$87 , $88,$89,$8A,$8B,$8C,$8D,$8E,$8F ; C |
| ; db $90,$91,$92,$93,$94,$95,$96,$97 , $98,$99,$9A,$9B,$9C,$9D,$9E,$9F ; D |
| ; db $A0,$A1,$A2,$A3,$A4,$A5,$A6,$A7 , $A8,$A9,$AA,$AB,$AC,$AD,$AE,$AF ; E |
| ; db $E0,$E1,$E2,$E3,$E4,$E5,$E6,$E7 , $E8,$E9,$EA,$EB,$EC,$ED,$EE,$EF ; F |
| |
| ; 0 1 2 3 4 5 6 7 8 9 A B C D E F |
| |
| utf8_table: |
| times 80h dw 0x98C3 ; default placeholder |
| |
| ; 0x80-0xAF -> 0x90D0-0xBFD0 |
| repeat 0x30 |
| store byte 0xD0 at utf8_table+2*(%-1) |
| store byte 0x90+%-1 at utf8_table+2*%-1 |
| end repeat |
| |
| ; 0xE0-0xEF -> 0x80D1-0x8FD1 |
| repeat 0x10 |
| store byte 0xD1 at utf8_table+2*(0xE0-0x80+%-1) |
| store byte 0x80+%-1 at utf8_table+2*(0xE0-0x80+%)-1 |
| end repeat |
| |
| ; 0xF0 -> 0x81D0, 0xF1 -> 0x91D1 |
| store dword 0x91D181D0 at utf8_table+2*(0xF0-0x80) |
| |
| ;cp866_table: |
| ; db $C0,$C1,$C2,$C3,$C4,$C5,$C6,$C7 , $C8,$C9,$CA,$CB,$CC,$CD,$CE,$CF ; 8 |
| ; db $D0,$D1,$D2,$D3,$D4,$D5,$D6,$D7 , $D8,$D9,$DA,$DB,$DC,$DD,$DE,$DF ; 9 |
| ; db $E0,$E1,$E2,$E3,$E4,$E5,$E6,$E7 , $E8,$E9,$EA,$EB,$EC,$ED,$EE,$EF ; A |
| ; db '?','?','?','?','?','?','?','?' , '?','?','?','?','?','?','?','?' ; B |
| ; db '?','?','?','?','?','?','?','?' , '?','?','?','?','?','?','?','?' ; C |
| ; db '?','?','?','?','?','?','?','?' , '?','?','?','?','?','?','?','?' ; D |
| ; db $F0,$F1,$F2,$F3,$F4,$F5,$F6,$F7 , $F8,$F9,$FA,$FB,$FC,$FD,$FE,$FF ; E |
| ; db $A8,$B8,$AA,$BA,$AF,$BF,$A1,$A2 , $B0,$95,$B7,'?',$B9,$A4,'?','?' ; F |
| |
| ; 0 1 2 3 4 5 6 7 8 9 A B C D E F |
| |
| |
| ; Codepoints for 0xB0-0xDF, unicode offset 0x2500 |
| cp866_boxes: |
| times 0xA0 db '?' |
| |
| store byte 0xB0 at cp866_boxes+0x91 |
| store byte 0xB1 at cp866_boxes+0x92 |
| store byte 0xB2 at cp866_boxes+0x93 |
| store byte 0xB3 at cp866_boxes+0x02 |
| store byte 0xB4 at cp866_boxes+0x24 |
| store byte 0xB5 at cp866_boxes+0x61 |
| store byte 0xB6 at cp866_boxes+0x62 |
| store byte 0xB7 at cp866_boxes+0x56 |
| |
| store byte 0xB8 at cp866_boxes+0x55 |
| store byte 0xB9 at cp866_boxes+0x63 |
| store byte 0xBA at cp866_boxes+0x51 |
| store byte 0xBB at cp866_boxes+0x57 |
| store byte 0xBC at cp866_boxes+0x5D |
| store byte 0xBD at cp866_boxes+0x5C |
| store byte 0xBE at cp866_boxes+0x5B |
| store byte 0xBF at cp866_boxes+0x10 |
| |
| store byte 0xC0 at cp866_boxes+0x14 |
| store byte 0xC1 at cp866_boxes+0x34 |
| store byte 0xC2 at cp866_boxes+0x2C |
| store byte 0xC3 at cp866_boxes+0x1C |
| store byte 0xC4 at cp866_boxes+0x00 |
| store byte 0xC5 at cp866_boxes+0x3C |
| store byte 0xC6 at cp866_boxes+0x5E |
| store byte 0xC7 at cp866_boxes+0x5F |
| |
| store byte 0xC8 at cp866_boxes+0x5A |
| store byte 0xC9 at cp866_boxes+0x54 |
| store byte 0xCA at cp866_boxes+0x69 |
| store byte 0xCB at cp866_boxes+0x66 |
| store byte 0xCC at cp866_boxes+0x60 |
| store byte 0xCD at cp866_boxes+0x50 |
| store byte 0xCE at cp866_boxes+0x6C |
| store byte 0xCF at cp866_boxes+0x67 |
| |
| store byte 0xD0 at cp866_boxes+0x68 |
| store byte 0xD1 at cp866_boxes+0x64 |
| store byte 0xD2 at cp866_boxes+0x65 |
| store byte 0xD3 at cp866_boxes+0x59 |
| store byte 0xD4 at cp866_boxes+0x58 |
| store byte 0xD5 at cp866_boxes+0x52 |
| store byte 0xD6 at cp866_boxes+0x53 |
| store byte 0xD7 at cp866_boxes+0x6B |
| |
| store byte 0xD8 at cp866_boxes+0x6A |
| store byte 0xD9 at cp866_boxes+0x18 |
| store byte 0xDA at cp866_boxes+0x0C |
| store byte 0xDB at cp866_boxes+0x88 |
| store byte 0xDC at cp866_boxes+0x84 |
| store byte 0xDD at cp866_boxes+0x8C |
| store byte 0xDE at cp866_boxes+0x90 |
| store byte 0xDF at cp866_boxes+0x80 |