mirror of
https://github.com/apple/swift.git
synced 2025-12-14 20:36:38 +01:00
313cfcaaa7
trie parameters and fix a few bugs The bugs did not affect correctness of the particular instance of trie created for grapheme cluster property, because trie parameters that were confused with each other happened to be equal. Also, fix a trie size bug: we were creating a trie large enough to store information for 0x200000 code points, but there are only 0x10ffff. It saved only 15 bytes in the grapheme cluster tree, because that extra information was compressed with some supplementary planes that also had default values. This also improved trie generation time by almost 2x. Swift SVN r19457
140 lines
4.7 KiB
Plaintext
140 lines
4.7 KiB
Plaintext
%{
|
|
|
|
# RUN: rm -rf %t && mkdir -p %t && %S/../../utils/gyb %s | FileCheck %s
|
|
#
|
|
# REQUIRES: long_tests
|
|
|
|
from GYBUnicodeDataUtils import *
|
|
|
|
def test_trie_generation(property_table, configure_generator=None):
|
|
trie_generator = UnicodeTrieGenerator()
|
|
if configure_generator is not None:
|
|
configure_generator(trie_generator)
|
|
trie_generator.create_tables()
|
|
trie_generator.fill_from_unicode_property(property_table)
|
|
trie_generator.verify(property_table)
|
|
trie_generator.freeze()
|
|
trie_generator.verify(property_table)
|
|
trie_generator.serialize(property_table)
|
|
print (
|
|
trie_generator.BMP_first_level_index_bits,
|
|
trie_generator.BMP_data_offset_bits,
|
|
trie_generator.supp_first_level_index_bits,
|
|
trie_generator.supp_second_level_index_bits,
|
|
trie_generator.supp_data_offset_bits,
|
|
|
|
trie_generator.BMP_lookup_bytes_per_entry,
|
|
trie_generator.BMP_data_bytes_per_entry,
|
|
trie_generator.supp_lookup1_bytes_per_entry,
|
|
trie_generator.supp_lookup2_bytes_per_entry,
|
|
trie_generator.supp_data_bytes_per_entry,
|
|
|
|
len(trie_generator.trie_bytes),
|
|
|
|
trie_generator.BMP_data_bytes_offset - trie_generator.BMP_lookup_bytes_offset,
|
|
trie_generator.supp_lookup1_bytes_offset - trie_generator.BMP_data_bytes_offset,
|
|
trie_generator.supp_lookup2_bytes_offset - trie_generator.supp_lookup1_bytes_offset,
|
|
trie_generator.supp_data_bytes_offset - trie_generator.supp_lookup2_bytes_offset,
|
|
len(trie_generator.trie_bytes) - trie_generator.supp_data_bytes_offset)
|
|
|
|
class PerfectlyCompressableProperty(UnicodeProperty):
|
|
def __init__(self):
|
|
pass
|
|
|
|
def get_default_value(self):
|
|
return 'Default'
|
|
|
|
def get_value(self, cp):
|
|
return 'Default'
|
|
|
|
def to_numeric_value(self, value):
|
|
if value == 'Default':
|
|
return 42
|
|
assert(False)
|
|
|
|
def get_numeric_value(self, cp):
|
|
return self.to_numeric_value(self.get_value(cp))
|
|
|
|
print 'PerfectlyCompressableProperty'
|
|
test_trie_generation(PerfectlyCompressableProperty())
|
|
# CHECK-LABEL: PerfectlyCompressableProperty
|
|
# CHECK: (8, 8, 5, 8, 8, 1, 1, 1, 1, 1, 1041, 256, 256, 17, 256, 256)
|
|
#
|
|
# Explanation for table sizes above:
|
|
#
|
|
# BMP_lookup: 1-byte words x 256 = 256
|
|
# BMP_data: 1 x 1 = 256
|
|
# supp_lookup1: 1 x 17 = 17
|
|
# supp_lookup2: 1 x 1*256 = 256
|
|
# supp_data: 1 x 1*256 = 256
|
|
|
|
|
|
class UncompressableProperty(UnicodeProperty):
|
|
def __init__(self):
|
|
pass
|
|
|
|
def get_default_value(self):
|
|
return 42
|
|
|
|
def get_value(self, cp):
|
|
# Split Unicode codespace into 128-entry "pages". Start each page with
|
|
# a unique sequence of property values (page number) so that the result
|
|
# can not be compressed.
|
|
page_number = cp >> 7
|
|
if cp % 0x80 == 1:
|
|
return page_number & 0xff
|
|
if cp % 0x80 == 2:
|
|
return (page_number >> 8) & 0xff
|
|
if cp % 0x80 == 3:
|
|
return (page_number >> 16) & 0xff
|
|
return 42
|
|
|
|
def to_numeric_value(self, value):
|
|
return value
|
|
|
|
def get_numeric_value(self, cp):
|
|
return self.to_numeric_value(self.get_value(cp))
|
|
|
|
print 'UncompressableProperty, default trie parameters'
|
|
test_trie_generation(UncompressableProperty())
|
|
# CHECK-LABEL: UncompressableProperty, default trie parameters
|
|
# CHECK: (8, 8, 5, 8, 8, 2, 1, 1, 2, 1, 1123601, 512, 65536, 17, 8704, 1048832)
|
|
#
|
|
# Explanation for table sizes above:
|
|
#
|
|
# BMP_lookup: 2-byte words x 256 = 512
|
|
# BMP_data: 1 x 256*256 = 65536
|
|
# supp_lookup1: 1 x 17 = 17
|
|
# supp_lookup2: 2 x 17*256 = 8704
|
|
# supp_data: 1 x (16*256+1)*256 = 1048832
|
|
|
|
def configure_generator_for_16_bit_indexes(trie_generator):
|
|
trie_generator.BMP_first_level_index_bits = 9
|
|
|
|
trie_generator.supp_first_level_index_bits = 10
|
|
trie_generator.supp_second_level_index_bits = 2
|
|
|
|
print 'UncompressableProperty, 16-bit indexes'
|
|
test_trie_generation(UncompressableProperty(),
|
|
configure_generator_for_16_bit_indexes)
|
|
# CHECK-LABEL: UncompressableProperty, 16-bit indexes
|
|
# CHECK: (9, 7, 10, 2, 9, 2, 1, 2, 2, 1, 1120840, 1024, 65536, 1088, 4104, 1049088)
|
|
#
|
|
# Explanation for table sizes above:
|
|
#
|
|
# BMP_lookup: 2-byte words x 512 = 1024
|
|
# BMP_data: 1 x 512*128 = 65536
|
|
# supp_lookup1: 2 x 544 = 1088
|
|
# supp_lookup2: 2 x 513*4 = 4104
|
|
# supp_data: 1 x (2048+1)*512 = 1049088
|
|
|
|
|
|
# gyb will print line markers after our output, so make sure that those
|
|
# don't accidentally match any other CHECK lines.
|
|
|
|
print 'THE END'
|
|
# CHECK-LABEL: THE END
|
|
|
|
}%
|
|
|