#
# (re)generate unicode property and type databases
#
# this script converts a unicode 3.2 database file to
# Modules/unicodedata_db.h, Modules/unicodename_db.h,
# and Objects/unicodetype_db.h
#
# history:
# 2000-09-24 fl   created (based on bits and pieces from unidb)
# 2000-09-25 fl   merged tim's splitbin fixes, separate decomposition table
# 2000-09-25 fl   added character type table
# 2000-09-26 fl   added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
# 2000-11-03 fl   expand first/last ranges
# 2001-01-19 fl   added character name tables (2.1)
# 2001-01-21 fl   added decomp compression; dynamic phrasebook threshold
# 2002-09-11 wd   use string methods
# 2002-10-18 mvl  update to Unicode 3.2
# 2002-10-22 mvl  generate NFC tables
# 2002-11-24 mvl  expand all ranges, sort names version-independently
# 2002-11-25 mvl  add UNIDATA_VERSION
# 2004-05-29 perky add east asian width information
# 2006-03-10 mvl  update to Unicode 4.1; add UCD 3.2 delta
# 2008-06-11 gb   add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
# 2011-10-21 ezio add support for name aliases and named sequences
# 2012-01    benjamin add full case mappings
#
# written by Fredrik Lundh (fredrik@pythonware.com)
#

import os
import sys

from textwrap import dedent

SCRIPT = sys.argv[0]
VERSION = "3.2"

# The Unicode Database
UNIDATA_VERSION = "7.0.0"
UNICODE_DATA = "UnicodeData.txt"
EASTASIAN_WIDTH = "EastAsianWidth.txt"
LINEBREAK_DATA = "LineBreak.txt"

CATEGORY_NAMES = [
    "Cn", "Ll", "Lm", "Lo", "Lt", "Lu",
    "Cf", "Co", "Cs",
    "Mc", "Me", "Mn", "Nd", "Nl", "No",
    "Pd", "Pe", "Pf", "Pi", "Po", "Ps",
    "Sk", "Sm", "So", "Zl", "Zp", "Zs" ]

BIDIRECTIONAL_NAMES = [ "ON", "L", "R", "AL", "NSM", "EN", "ES", "ET", "AN", "CS", "BN", "B", "S", "WS",
                        "LRE", "LRI", "LRO", "RLE", "RLI", "RLO", "PDI", "PDF", "FSI"]

#EASTASIANWIDTH_NAMES = [ "N", "F", "W", "A", "Na", "H" ]
EASTASIANWIDTH_NAMES = [ "N", "F", "W", "A", "Na" ]

LINEBREAK_NAMES = [ "AL", "AI", "ID", "SG", "CM", "OP", "CP", "JL", "JV", "JT",
                    "CL", "GL", "QU", "CJ", "NS", "EX", "IS", "PR", "PO", "NU", "IN", "BB", "BA", "HL",
                    "RI", "B2", "BK", "WJ", "CB", "SP", "ZW", "LF", "NL", "HY", "SY", "SA", "H2", "H3" ]

# note: should match definitions in Objects/unicodectype.c
ALPHA_MASK = 0x01
UPPER_MASK = 0x02
LOWER_MASK = 0x04
TITLE_MASK = 0x08
LINEBREAK_MASK = 0x10
SPACE_MASK = 0x20
DIGIT_MASK = 0x40
PRINTABLE_MASK = 0x80
CASE_IGNORABLE_MASK = 0x100
CASED_MASK = 0x200
EXTENDED_CASE_MASK = 0x400

def maketables(trace=0):

    print("--- Reading", UNICODE_DATA, "...")

    version = ""
    unicode = UnicodeData(UNIDATA_VERSION)

    print(len(list(filter(None, unicode.table))), "characters")

    makeunicodedata(unicode, trace)

# --------------------------------------------------------------------
# unicode character properties

def makeunicodedata(unicode, trace):

    dummy = (CATEGORY_NAMES.index("Cn"), 0, EASTASIANWIDTH_NAMES.index("N"))
    table = [dummy]
    cache = {0: dummy}
    index= [0] * len(unicode.chars)
    breakstats = [0] * 40
    categorystats = [0] * 64
    categorystats2 = [0] * 64

    FILE = "unicodeprops_db.h"

    print("--- Preparing", FILE, "...")
    item = (0, 0, 0)
    cache[item] = 0
    table.append(item)
    item = (7, 3, 0)
    cache[item] = 1
    table.append(item)

    # 1) database properties

    for char in unicode.chars:
        record = unicode.table[char]
        if record:
            # extract database properties
            if record[2] == "Nd":
                if char > 0x2f and char < 0x3a:
                    pass
                elif char > 0xff0f and char < 0xff1a:
                    pass
                elif char > 0x1d7cd and char < 0x1d800:
                    pass
                else:
                    record[2] = "Cn"
            if char >= 0x3400 and char < 0x4db6:
                record[2] = "Lo"
            if char >= 0x4e00 and char < 0xa000:
                record[2] = "Lo"
            category = CATEGORY_NAMES.index(record[2])
            if char < 0x10000:
                categorystats[category] = categorystats[category] + 1
            else:
                categorystats2[category] = categorystats2[category] + 1
            combining = int(record[3])
            eastasianwidth = EASTASIANWIDTH_NAMES.index(record[15])
            # XXX: not used at the moment
            item = (category, combining, eastasianwidth)
            # add entry to index and item tables
            i = cache.get(item)
            if i is None:
                cache[item] = i = len(table)
                table.append(item)
            index[char] = i

    print("--- Character property type occurences: (BMP, SMP)")
    for i in range(0, 27):
        print ("%s (%d): %d, %d" % (CATEGORY_NAMES[i], i, categorystats[i], categorystats2[i]))

    print(len(table), "unique properties")

    print("--- Writing", FILE, "...")

    fp = open(FILE, "w")
    print("/* this file was generated by %s %s */" % (SCRIPT, VERSION), file=fp)
    print('#define UNIDATA_VERSION "%s"' % UNIDATA_VERSION, file=fp)
    print(file=fp)
    print('typedef struct {\n    unsigned char category;\n    unsigned char combining;\n    unsigned char east_asian_width;\n} UCDRecord;', file=fp)
    print(file=fp)
    print("/* a list of unique database records */", file=fp)
    print("static const UCDRecord ucd_records[] = {", file=fp)
    for item in table:
        print("    {%d, %d,  %d}," % item, file=fp)
    print("};", file=fp)
    print(file=fp)

    for name in EASTASIANWIDTH_NAMES:
        print("#define UCDN_EAST_ASIAN_%s %d" % (name.upper(), EASTASIANWIDTH_NAMES.index(name)), file=fp)
    print(file=fp)

    for name in CATEGORY_NAMES:
        print("#define UCDN_GENERAL_CATEGORY_%s %d" % (name.upper(), CATEGORY_NAMES.index(name)), file=fp)
    print(file=fp)

    """
    print("static const char *decomp_prefix[] = {", file=fp)
    for name in decomp_prefix:
        print("    \"%s\"," % name, file=fp)
    print("    NULL", file=fp)
    print("};", file=fp)
    """

    # split record index table
    index1, index2, shift = splitbins(index, trace)
    print("/* index tables for the database records */", file=fp)
    print("#define SHIFT", shift, file=fp)
    Array("index1", index1).dump(fp, trace)
    Array("index2", index2).dump(fp, trace)

    fp.close()

def open_data(local, version):
    return open(local, encoding='utf-8')

# --------------------------------------------------------------------
# the following support code is taken from the unidb utilities
# Copyright (c) 1999-2000 by Secret Labs AB

# load a unicode-data file from disk

class UnicodeData:
    # Record structure:
    # [ID, name, category, combining, bidi, decomp,  (6)
    #  decimal, digit, numeric, bidi-mirrored, Unicode-1-name, (11)
    #  ISO-comment, uppercase, lowercase, titlecase, ea-width, (16)
    #  derived-props] (17)

    def __init__(self, version,
                 linebreakprops=False,
                 expand=1):
        self.changed = []
        table = [None] * 0x20000
        with open_data(UNICODE_DATA, version) as file:
            while 1:
                s = file.readline()
                if not s:
                    break
                s = s.strip().split(";")
                char = int(s[0], 16)
                table[char] = s

        for i in range(0, 0x20000):
            if table[i] is None:
                table[i] = ["", "UNASSIGNED CODEPOINT", "Cn", "0", "ON", "", "", "", "", "N", "", "", "", "", ""]

        # expand first-last ranges
        if expand:
            field = None
            for i in range(0, 0x20000):
                s = table[i]
                if s:
                    if s[1][-6:] == "First>":
                        s[1] = ""
                        field = s
                    elif s[1][-5:] == "Last>":
                        s[1] = ""
                        field = None
                    elif field:
                        f2 = field[:]
                        f2[0] = "%X" % i
                        table[i] = f2
                elif field:
                    f2 = field[:]
                    f2[0] = "%X" % i
                    table[i] = f2

        # public attributes
        self.filename = UNICODE_DATA
        self.table = table
        self.chars = list(range(0x20000)) # unicode 3.2

        widths = ["N"] * 0x20000
        with open_data(EASTASIAN_WIDTH, version) as file:
            for s in file:
                s = s.strip()
                if not s:
                    continue
                if s[0] == '#':
                    continue
                s = s.split()[0].split(';')
                if '..' in s[0]:
                    first, last = [int(c, 16) for c in s[0].split('..')]
                    chars = list(range(first, last+1))
                else:
                    chars = [int(s[0], 16)]
                for char in chars:
                    widths[char] = s[1]

        for i in range(0, 0x20000):
            if table[i] is not None:
                table[i].append(widths[i])

# hash table tools

# this is a straight-forward reimplementation of Python's built-in
# dictionary type, using a static data structure, and a custom string
# hash algorithm.

def myhash(s, magic):
    h = 0
    for c in map(ord, s.upper()):
        h = (h * magic) + c
        ix = h & 0xff000000
        if ix:
            h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff
    return h

SIZES = [
    (4,3), (8,3), (16,3), (32,5), (64,3), (128,3), (256,29), (512,17),
    (1024,9), (2048,5), (4096,83), (8192,27), (16384,43), (32768,3),
    (65536,45), (131072,9), (262144,39), (524288,39), (1048576,9),
    (2097152,5), (4194304,3), (8388608,33), (16777216,27)
]

class Hash:
    def __init__(self, name, data, magic):
        # turn a (key, value) list into a static hash table structure

        # determine table size
        for size, poly in SIZES:
            if size > len(data):
                poly = size + poly
                break
        else:
            raise AssertionError("ran out of polynomials")

        print(size, "slots in hash table")

        table = [None] * size

        mask = size-1

        n = 0

        hash = myhash

        # initialize hash table
        for key, value in data:
            h = hash(key, magic)
            i = (~h) & mask
            v = table[i]
            if v is None:
                table[i] = value
                continue
            incr = (h ^ (h >> 3)) & mask;
            if not incr:
                incr = mask
            while 1:
                n = n + 1
                i = (i + incr) & mask
                v = table[i]
                if v is None:
                    table[i] = value
                    break
                incr = incr << 1
                if incr > mask:
                    incr = incr ^ poly

        print(n, "collisions")
        self.collisions = n

        for i in range(len(table)):
            if table[i] is None:
                table[i] = 0

        self.data = Array(name + "_hash", table)
        self.magic = magic
        self.name = name
        self.size = size
        self.poly = poly

    def dump(self, file, trace):
        # write data to file, as a C array
        self.data.dump(file, trace)
        file.write("#define %s_magic %d\n" % (self.name, self.magic))
        file.write("#define %s_size %d\n" % (self.name, self.size))
        file.write("#define %s_poly %d\n" % (self.name, self.poly))

# stuff to deal with arrays of unsigned integers

class Array:

    def __init__(self, name, data):
        self.name = name
        self.data = data

    def dumphex(self, file, trace=0):
        # write data to file, as a C array
        size = getsize(self.data)
        if trace:
            print(self.name+":", size*len(self.data), "bytes", file=sys.stderr)
        file.write("static const unsigned short ")
        file.write(" " + self.name + "[%d] = {\n" % (len(self.data)))
        k = 0
        if self.data:
            s = "    "
            for item in self.data:
                i = "0x%04x, " % item
                if k > 7:
                    file.write(s + "\n")
                    s = "    " + i
                    k = 0
                else:
                    s = s + i
                k = k + 1
            if s.strip():
                file.write(s + "\n")
        file.write("};\n\n")

    def dump(self, file, trace=0):
        # write data to file, as a C array
        size = getsize(self.data)
        if trace:
            print(self.name+":", size*len(self.data), "bytes", file=sys.stderr)
        file.write("static const ")
        if size == 1:
            file.write("unsigned char")
        elif size == 2:
            file.write("unsigned short")
        else:
            file.write("unsigned int")
        offset = 0
        file.write(" " + self.name + "[%d] = {" % (len(self.data)))
        if self.data:
            s = "    "
            for item in self.data:
                #if len(s) + len(i) > 78:
                #    file.write(s + "\n")
                #    s = "    " + i
                i = "%4u, " % item
                if (offset % 16) == 0:
                    file.write(s + "\n")
                    s = "   " + i
                else:
                    s = s + i
                offset = offset + 1
            if s.strip():
                file.write(s + "\n")
        file.write("};\n\n")

def getsize(data):
    # return smallest possible integer size for the given array
    maxdata = max(data)
    if maxdata < 256:
        return 1
    elif maxdata < 65536:
        return 2
    else:
        return 4

def splitbins(t, trace=0):
    """t, trace=0 -> (t1, t2, shift).  Split a table to save space.

    t is a sequence of ints.  This function can be useful to save space if
    many of the ints are the same.  t1 and t2 are lists of ints, and shift
    is an int, chosen to minimize the combined size of t1 and t2 (in C
    code), and where for each i in range(len(t)),
        t[i] == t2[(t1[i >> shift] << shift) + (i & mask)]
    where mask is a bitmask isolating the last "shift" bits.

    If optional arg trace is non-zero (default zero), progress info
    is printed to sys.stderr.  The higher the value, the more info
    you'll get.
    """

    def dump(t1, t2, shift, bytes):
        print("%d+%d bins at shift %d; %d bytes" % (
            len(t1), len(t2), shift, bytes), file=sys.stderr)
    print("Size of original table:", len(t)*getsize(t), \
                            "bytes", file=sys.stderr)
    n = len(t)-1    # last valid index
    tablen = n
    while not t[tablen] or t[tablen] == 0:
        tablen = tablen - 1
    print("Size of original table:", tablen*getsize(t), \
                            "bytes", file=sys.stderr)
    tablen = n
    maxshift = 0    # the most we can shift n and still have something left
    if n > 0:
        while n >> 1:
            n >>= 1
            maxshift += 1
    del n
    bytes = sys.maxsize  # smallest total size so far
    t = tuple(t)    # so slices can be dict keys
    for shift in range(maxshift + 1):
        t1 = []
        t2 = []
        size = 2**shift
        bincache = {}
        for i in range(0, len(t), size):
            bin = t[i:i+size]
            index = bincache.get(bin)
            if index is None:
                index = len(t2)
                bincache[bin] = index
                t2.extend(bin)
            t1.append(index >> shift)
        # determine memory size
        b = len(t1)*getsize(t1) + len(t2)*getsize(t2)
        dump(t1, t2, shift, b)
        if b < bytes:
            best = t1, t2, shift
            bytes = b
    t1, t2, shift = best
    print("Best:", end=' ', file=sys.stderr)
    dump(t1, t2, shift, bytes)
    return best

def encode_utf16_surrogates(code):
    assert(code < 0x110000)
    code = code - 0x10000
    low = code & 1023
    high = code >> 10
    high_sur = 0xd800 + high
    low_sur = 0xdc00 + low
    return high_sur, low_sur

if __name__ == "__main__":
    maketables(1)