-- FILE:     UTF8.E
-- PURPOSE:  Encode/Decode Unicode to/from UTF-8.
-- AUTHOR:   Shian Lee
-- VERSION:  2.10  December/27/2021
-- LANGUAGE: Euphoria version 3.1.1 (http://www.RapidEuphoria.com)
-- PLATFORM: Any
-- LICENCE:  Free. Use at your own risk.
-- NOTE:     * See the attached reference manual UTF8.DOC for details.
--           * Created using QE v2.3.9 for DOS (RapidEuphoria.com archive).
--           * Basic test: https://www.cl.cam.ac.uk/~mgk25/ucs/full-bmp.txt
--           * Unicode code points range for the UTF-8 encoding (1..4 bytes):
-- ___________________________________________________________________________
-- |Bytes |    First..Last code point    |  head  |  tail  |  tail  |  tail  |
-- |(Bits)| Hex           | Dec          | Byte 1 | Byte 2 | Byte 3 | Byte 4 |
-- |~~~~~~|~~~~~~~~~~~~~~~|~~~~~~~~~~~~~~|~~~~~~~~|~~~~~~~~|~~~~~~~~|~~~~~~~~|
-- |1  (7)|#0..#7F        |0..127        |0xxxxxxx|        |        |        |
-- |2 (11)|#80..#7FF      |128..2047     |110xxxxx|10xxxxxx|        |        |
-- |3 (16)|#800..#FFFF    |2048..65535   |1110xxxx|10xxxxxx|10xxxxxx|        |
-- |4 (21)|#10000..#10FFFF|65536..1114111|11110xxx|10xxxxxx|10xxxxxx|10xxxxxx|
-- ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
--  - 4 bytes encoding is limited to #10FFFF (not #1FFFFF).
--  - connect the x's of all bytes to get the Unicode code point.
--
-- HISTORY:  v1.01 - initial version
--           v2.00 - rewriting code, adding standard user manual, demos, and
--                   additional include files.
--           v2.01 - minor bug fix in unicode() function, verify tail bytes.
--           v2.02 - utf8() accepts object. UTF8_REP/UNICODE_REP are global.
--           v2.10 - utf8() named unicode_to_utf8(); 
--                   unicode() named utf8_to_unicode(); -- longer but clear.
-- --------------------------------------------------------------------------



--bookmark: include library files;
include binconst.e      -- binary constants



--bookmark: Local Constants;

constant
    EMPTY = {},

    SHx18 = #40000,  -- power(2, 18),  shift 18-bits
    SHx12 = #1000,   -- power(2, 12),  shift 12-bits
    SHx6  = #40,     -- power(2, 6),   shift 6-bits

    -- min/max range per bytes (see the table above)
    MIN1 = #0,
    MIN2 = #80,
    MIN3 = #800,
    MIN4 = #10000,

    MAX1 = #7F,
    MAX2 = #7FF,
    MAX3 = #FFFF,
    MAX4 = #10FFFF



--bookmark: Decoding/Encoding;

global constant
    -- the byte-order-mark (BOM) UTF-8 character/Unicode code point
       UTF8_BOM = {#EF, #BB, #BF},
    UNICODE_BOM = #FEFF,

    -- the replacement-character UTF-8 character/Unicode code point
       UTF8_REP = {#EF, #BF, #BD},
    UNICODE_REP = #FFFD



-- PURPOSE: decode UTF-8 string into Unicode string
-- DETAILS: s is a string-sequence of UTF-8 characters
-- RETURN:  string; Unicode code points
--          * invalid or long-form UTF-8 characters returned as UNICODE_REP
-- EXAMPLE: s = utf8_to_unicode({-5, 127, 223, 191})
--            --> s is {UNICODE_REP, 127, #7FF}
global function utf8_to_unicode(sequence s)
    sequence uni
    integer len, i, head, bytes, code

    -- note: the basic algorithm is copied to group_utf8(),
    --       any change should be applied to both functions.

    uni = EMPTY
    len = length(s)
    i = 1
    while i <= len do
	-- head (first) byte of UTF-8 character
	head = s[i]

	-- calculate length of UTF-8 character in bytes (1..4):
	if    head <  B00000000 then bytes = 0  -- < MIN1
	elsif head <= B01111111 then bytes = 1  -- head: 0xxxxxxx
	elsif head <= B10111111 then bytes = 0  -- tail: 10xxxxxx (illegal)
	elsif head <= B11011111 then bytes = 2  -- head: 110xxxxx
	elsif head <= B11101111 then bytes = 3  -- head: 1110xxxx
	elsif head <= B11110100 then bytes = 4  -- head: 11110xzz
	else -- > MAX4
	    bytes = 0
	end if

	-- verify tail bytes (2,3,4)
	if bytes > 1 then
	    if i + bytes - 1 > len then
		bytes = 0                       -- end of string too soon
	    else
		for j = 1 to bytes - 1 do
		    if s[i + j] < B10000000 or s[i + j] > B10111111 then
			bytes = 0
			exit                    -- tail byte out of range
		    end if
		end for
	    end if
	end if

	-- pack the 'x' bits to create Unicode code point
	if bytes = 1 then                   -- head: 0xxxxxxx (ASCII)
	    code = head

	elsif bytes = 2 then                -- head: 110xxxxx
	    code = and_bits(head, B00011111) * SHx6 +
		and_bits(s[i + 1], B00111111)

	    if code < MIN2 then             -- long form?
		code = UNICODE_REP
	    end if

	elsif bytes = 3 then                -- head: 1110xxxx
	    code = and_bits(head, B00001111) * SHx12 +
		and_bits(s[i + 1], B00111111) * SHx6 +
		and_bits(s[i + 2], B00111111)

	    if code < MIN3 then             -- long form?
		code = UNICODE_REP
	    end if

	elsif bytes = 4 then                -- head: 11110xzz
	    code = and_bits(head, B00000111) * SHx18 +
		and_bits(s[i + 1], B00111111) * SHx12 +
		and_bits(s[i + 2], B00111111) * SHx6 +
		and_bits(s[i + 3], B00111111)

	    if code < MIN4 or code > MAX4 then  -- long form or > MAX4?
		code = UNICODE_REP
	    end if

	else -- bytes = 0 (invalid UTF-8 byte), move to next UTF-8 byte
	    code = UNICODE_REP
	    i += 1
	end if

	-- append Unicode code point, move to next UTF-8 character
	uni &= code
	i += bytes
    end while

    return uni
end function



-- PURPOSE: encode Unicode string into UTF-8 string
-- DETAILS: x is an integer/string-sequence of Unicode code points
-- RETURN:  string; UTF-8 characters
--          * invalid Unicode code points returned as UTF8_REP
-- EXAMPLE: s = unicode_to_utf8({-5, #7F, #10FFFF, #110000})
--            --> s is {UTF8_REP, 127, 244, 143, 191, 191, UTF8_REP}
constant
    TAILS_2 = {SHx6, 1},            -- factor: unpack 2 tails
    TAILS_3 = {SHx12, SHx6, 1}      -- factor: unpack 3 tails
global function unicode_to_utf8(object x)
    sequence s, utf
    integer code

    if atom(x) then s = {x} else s = x end if
    utf = EMPTY

    for i = 1 to length(s) do
	-- Unicode code point
	code = s[i]

	-- unpack the 'x' bits to create UTF-8 character
	if code < MIN1 then                 -- invalid code point
	    utf &= UTF8_REP

	elsif code <= MAX1 then             -- head: 0xxxxxxx (ASCII)
	    utf &= code

	elsif code <= MAX2 then             -- head: 110xxxxx
	    utf &= or_bits(floor(code / SHx6), B11000000) &
		or_bits(B10000000, and_bits(B00111111, code))

	elsif code <= MAX3 then             -- head: 1110xxxx
	    utf &= or_bits(floor(code / SHx12), B11100000) &
		or_bits(B10000000, and_bits(B00111111, floor(code / TAILS_2)))

	elsif code <= MAX4 then             -- head: 11110xzz
	    utf &= or_bits(floor(code / SHx18), B11110000) &
		or_bits(B10000000, and_bits(B00111111, floor(code / TAILS_3)))

	else -- code > MAX4 (invalid code point)
	    utf &= UTF8_REP
	end if
    end for

    return utf
end function



-- PURPOSE: group UTF-8 characters as sub-strings
-- DETAILS: s is a string-sequence of UTF-8 characters
-- RETURN:  sequence; grouped UTF-8 characters
--          * invalid UTF-8 characters returned as UTF8_REP
-- EXAMPLE: s = group_utf8({-5, 127, 223, 191})
--            --> s is {UTF8_REP, 127, {223, 191}}
global function group_utf8(sequence s)
    sequence utf
    integer len, i, head, bytes

    -- note: the basic algorithm is copied from utf8_to_unicode(),
    --       any change should be applied to both functions.

    utf = EMPTY
    len = length(s)
    i = 1
    while i <= len do
	-- head (first) byte of UTF-8 character
	head = s[i]

	-- calculate length of UTF-8 character in bytes (1..4):
	if    head <  B00000000 then bytes = 0  -- < MIN1
	elsif head <= B01111111 then bytes = 1  -- head: 0xxxxxxx
	elsif head <= B10111111 then bytes = 0  -- tail: 10xxxxxx (illegal)
	elsif head <= B11011111 then bytes = 2  -- head: 110xxxxx
	elsif head <= B11101111 then bytes = 3  -- head: 1110xxxx
	elsif head <= B11110100 then bytes = 4  -- head: 11110xzz
	else -- > MAX4
	    bytes = 0
	end if

	-- verify tail bytes (2,3,4)
	if bytes > 1 then
	    if i + bytes - 1 > len then
		bytes = 0                       -- end of string too soon
	    else
		for j = 1 to bytes - 1 do
		    if s[i + j] < B10000000 or s[i + j] > B10111111 then
			bytes = 0
			exit                    -- tail byte out of range
		    end if
		end for
	    end if
	end if

	-- group bytes of UTF-8 character into sub-string (don't group ASCII)
	if bytes = 1 then                   -- ASCII byte
	    utf &= head
	elsif bytes >= 2 then               -- 2,3,4 bytes
	    utf = append(utf, s[i..i + bytes - 1])
	else -- bytes = 0 (invalid UTF-8 byte), move to next UTF-8 byte
	    utf = append(utf, UTF8_REP)
	    i += 1
	end if

	-- move to next UTF-8 character
	i += bytes
    end while

    return utf
end function



-- end of file.
