Skip to content

Commit

Permalink
bpo-37760: Factor out the basic UCD parsing logic of makeunicodedata. (…
Browse files Browse the repository at this point in the history 
…pythonGH-15130)

There were 10 copies of this, and almost as many distinct versions of
exactly how it was written.  They're all implementing the same
standard.  Pull them out to the top, so the more interesting logic
that remains becomes easier to read.
  • Loading branch information
@gnprice @benjaminp
gnprice authored and benjaminp committed Aug 13, 2019
1 parent 66a34d3 commit ef2af1a
Showing 1 changed file with 109 additions and 133 deletions.
242 changes: 109 additions & 133 deletions Tools/unicode/makeunicodedata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@
import sys
import zipfile

from textwrap import dedent
from functools import partial
from textwrap import dedent
from typing import *

SCRIPT = sys.argv[0]
VERSION = "3.3"
Expand Down Expand Up @@ -903,6 +904,32 @@ def open_data(template, version):
return open(local, 'rb')


class UcdFile:
'''
A file in the standard format of the UCD.
See: https://www.unicode.org/reports/tr44/#Format_Conventions
Note that, as described there, the Unihan data files have their
own separate format.
'''

def __init__(self, template: str, version: str) -> None:
self.template = template
self.version = version

def records(self) -> Iterator[List[str]]:
with open_data(self.template, self.version) as file:
for line in file:
line = line.split('#', 1)[0].strip()
if not line:
continue
yield [field.strip() for field in line.split(';')]

def __iter__(self) -> Iterator[List[str]]:
return self.records()


# --------------------------------------------------------------------
# the following support code is taken from the unidb utilities
# Copyright (c) 1999-2000 by Secret Labs AB
Expand All @@ -922,14 +949,9 @@ def __init__(self, version,
cjk_check=True):
self.changed = []
table = [None] * 0x110000
with open_data(UNICODE_DATA, version) as file:
while 1:
s = file.readline()
if not s:
break
s = s.strip().split(";")
char = int(s[0], 16)
table[char] = s
for s in UcdFile(UNICODE_DATA, version):
char = int(s[0], 16)
table[char] = s

cjk_ranges_found = []

Expand Down Expand Up @@ -968,17 +990,12 @@ def __init__(self, version,
# in order to take advantage of the compression and lookup
# algorithms used for the other characters
pua_index = NAME_ALIASES_START
with open_data(NAME_ALIASES, version) as file:
for s in file:
s = s.strip()
if not s or s.startswith('#'):
continue
char, name, abbrev = s.split(';')
char = int(char, 16)
self.aliases.append((name, char))
# also store the name in the PUA 1
self.table[pua_index][1] = name
pua_index += 1
for char, name, abbrev in UcdFile(NAME_ALIASES, version):
char = int(char, 16)
self.aliases.append((name, char))
# also store the name in the PUA 1
self.table[pua_index][1] = name
pua_index += 1
assert pua_index - NAME_ALIASES_START == len(self.aliases)

self.named_sequences = []
Expand All @@ -988,50 +1005,32 @@ def __init__(self, version,

assert pua_index < NAMED_SEQUENCES_START
pua_index = NAMED_SEQUENCES_START
with open_data(NAMED_SEQUENCES, version) as file:
for s in file:
s = s.strip()
if not s or s.startswith('#'):
continue
name, chars = s.split(';')
chars = tuple(int(char, 16) for char in chars.split())
# check that the structure defined in makeunicodename is OK
assert 2 <= len(chars) <= 4, "change the Py_UCS2 array size"
assert all(c <= 0xFFFF for c in chars), ("use Py_UCS4 in "
"the NamedSequence struct and in unicodedata_lookup")
self.named_sequences.append((name, chars))
# also store these in the PUA 1
self.table[pua_index][1] = name
pua_index += 1
for name, chars in UcdFile(NAMED_SEQUENCES, version):
chars = tuple(int(char, 16) for char in chars.split())
# check that the structure defined in makeunicodename is OK
assert 2 <= len(chars) <= 4, "change the Py_UCS2 array size"
assert all(c <= 0xFFFF for c in chars), ("use Py_UCS4 in "
"the NamedSequence struct and in unicodedata_lookup")
self.named_sequences.append((name, chars))
# also store these in the PUA 1
self.table[pua_index][1] = name
pua_index += 1
assert pua_index - NAMED_SEQUENCES_START == len(self.named_sequences)

self.exclusions = {}
with open_data(COMPOSITION_EXCLUSIONS, version) as file:
for s in file:
s = s.strip()
if not s:
continue
if s[0] == '#':
continue
char = int(s.split()[0],16)
self.exclusions[char] = 1
for char, in UcdFile(COMPOSITION_EXCLUSIONS, version):
char = int(char, 16)
self.exclusions[char] = 1

widths = [None] * 0x110000
with open_data(EASTASIAN_WIDTH, version) as file:
for s in file:
s = s.strip()
if not s:
continue
if s[0] == '#':
continue
s = s.split()[0].split(';')
if '..' in s[0]:
first, last = [int(c, 16) for c in s[0].split('..')]
chars = list(range(first, last+1))
else:
chars = [int(s[0], 16)]
for char in chars:
widths[char] = s[1]
for s in UcdFile(EASTASIAN_WIDTH, version):
if '..' in s[0]:
first, last = [int(c, 16) for c in s[0].split('..')]
chars = list(range(first, last+1))
else:
chars = [int(s[0], 16)]
for char in chars:
widths[char] = s[1]

for i in range(0, 0x110000):
if table[i] is not None:
Expand All @@ -1041,38 +1040,27 @@ def __init__(self, version,
if table[i] is not None:
table[i].append(set())

with open_data(DERIVED_CORE_PROPERTIES, version) as file:
for s in file:
s = s.split('#', 1)[0].strip()
if not s:
continue

r, p = s.split(";")
r = r.strip()
p = p.strip()
if ".." in r:
first, last = [int(c, 16) for c in r.split('..')]
chars = list(range(first, last+1))
else:
chars = [int(r, 16)]
for char in chars:
if table[char]:
# Some properties (e.g. Default_Ignorable_Code_Point)
# apply to unassigned code points; ignore them
table[char][-1].add(p)

with open_data(LINE_BREAK, version) as file:
for s in file:
s = s.partition('#')[0]
s = [i.strip() for i in s.split(';')]
if len(s) < 2 or s[1] not in MANDATORY_LINE_BREAKS:
continue
if '..' not in s[0]:
first = last = int(s[0], 16)
else:
first, last = [int(c, 16) for c in s[0].split('..')]
for char in range(first, last+1):
table[char][-1].add('Line_Break')
for r, p in UcdFile(DERIVED_CORE_PROPERTIES, version):
if ".." in r:
first, last = [int(c, 16) for c in r.split('..')]
chars = list(range(first, last+1))
else:
chars = [int(r, 16)]
for char in chars:
if table[char]:
# Some properties (e.g. Default_Ignorable_Code_Point)
# apply to unassigned code points; ignore them
table[char][-1].add(p)

for s in UcdFile(LINE_BREAK, version):
if len(s) < 2 or s[1] not in MANDATORY_LINE_BREAKS:
continue
if '..' not in s[0]:
first = last = int(s[0], 16)
else:
first, last = [int(c, 16) for c in s[0].split('..')]
for char in range(first, last+1):
table[char][-1].add('Line_Break')

# We only want the quickcheck properties
# Format: NF?_QC; Y(es)/N(o)/M(aybe)
Expand All @@ -1083,23 +1071,19 @@ def __init__(self, version,
# for older versions, and no delta records will be created.
quickchecks = [0] * 0x110000
qc_order = 'NFD_QC NFKD_QC NFC_QC NFKC_QC'.split()
with open_data(DERIVEDNORMALIZATION_PROPS, version) as file:
for s in file:
if '#' in s:
s = s[:s.index('#')]
s = [i.strip() for i in s.split(';')]
if len(s) < 2 or s[1] not in qc_order:
continue
quickcheck = 'MN'.index(s[2]) + 1 # Maybe or No
quickcheck_shift = qc_order.index(s[1])*2
quickcheck <<= quickcheck_shift
if '..' not in s[0]:
first = last = int(s[0], 16)
else:
first, last = [int(c, 16) for c in s[0].split('..')]
for char in range(first, last+1):
assert not (quickchecks[char]>>quickcheck_shift)&3
quickchecks[char] |= quickcheck
for s in UcdFile(DERIVEDNORMALIZATION_PROPS, version):
if len(s) < 2 or s[1] not in qc_order:
continue
quickcheck = 'MN'.index(s[2]) + 1 # Maybe or No
quickcheck_shift = qc_order.index(s[1])*2
quickcheck <<= quickcheck_shift
if '..' not in s[0]:
first = last = int(s[0], 16)
else:
first, last = [int(c, 16) for c in s[0].split('..')]
for char in range(first, last+1):
assert not (quickchecks[char]>>quickcheck_shift)&3
quickchecks[char] |= quickcheck
for i in range(0, 0x110000):
if table[i] is not None:
table[i].append(quickchecks[i])
Expand All @@ -1122,34 +1106,26 @@ def __init__(self, version,
# Patch the numeric field
if table[i] is not None:
table[i][8] = value

sc = self.special_casing = {}
with open_data(SPECIAL_CASING, version) as file:
for s in file:
s = s[:-1].split('#', 1)[0]
if not s:
continue
data = s.split("; ")
if data[4]:
# We ignore all conditionals (since they depend on
# languages) except for one, which is hardcoded. See
# handle_capital_sigma in unicodeobject.c.
continue
c = int(data[0], 16)
lower = [int(char, 16) for char in data[1].split()]
title = [int(char, 16) for char in data[2].split()]
upper = [int(char, 16) for char in data[3].split()]
sc[c] = (lower, title, upper)
for data in UcdFile(SPECIAL_CASING, version):
if data[4]:
# We ignore all conditionals (since they depend on
# languages) except for one, which is hardcoded. See
# handle_capital_sigma in unicodeobject.c.
continue
c = int(data[0], 16)
lower = [int(char, 16) for char in data[1].split()]
title = [int(char, 16) for char in data[2].split()]
upper = [int(char, 16) for char in data[3].split()]
sc[c] = (lower, title, upper)

cf = self.case_folding = {}
if version != '3.2.0':
with open_data(CASE_FOLDING, version) as file:
for s in file:
s = s[:-1].split('#', 1)[0]
if not s:
continue
data = s.split("; ")
if data[1] in "CF":
c = int(data[0], 16)
cf[c] = [int(char, 16) for char in data[2].split()]
for data in UcdFile(CASE_FOLDING, version):
if data[1] in "CF":
c = int(data[0], 16)
cf[c] = [int(char, 16) for char in data[2].split()]

def uselatin1(self):
# restrict character range to ISO Latin 1
Expand Down

0 comments on commit ef2af1a

Please sign in to comment.