#! /usr/bin/python # PCRE2 UNICODE PROPERTY SUPPORT # ------------------------------ # This file is a Python module containing common lists and functions for the # GenerateXXX scripts that create various.c and .h files from Unicode data # files. It was created as part of a re-organizaton of these scripts in # December 2021. import re # --------------------------------------------------------------------------- # DATA LISTS # --------------------------------------------------------------------------- # BIDI classes in the DerivedBidiClass.txt file, with comments. bidi_classes = [ 'AL', 'Arabic letter', 'AN', 'Arabic number', 'B', 'Paragraph separator', 'BN', 'Boundary neutral', 'CS', 'Common separator', 'EN', 'European number', 'ES', 'European separator', 'ET', 'European terminator', 'FSI', 'First strong isolate', 'L', 'Left to right', 'LRE', 'Left to right embedding', 'LRI', 'Left to right isolate', 'LRO', 'Left to right override', 'NSM', 'Non-spacing mark', 'ON', 'Other neutral', 'PDF', 'Pop directional format', 'PDI', 'Pop directional isolate', 'R', 'Right to left', 'RLE', 'Right to left embedding', 'RLI', 'Right to left isolate', 'RLO', 'Right to left override', 'S', 'Segment separator', 'WS', 'White space' ] # Particular category property names, with comments. NOTE: If ever this list # is changed, the table called "catposstab" in the pcre2_auto_possess.c file # must be edited to keep in step. category_names = [ 'Cc', 'Control', 'Cf', 'Format', 'Cn', 'Unassigned', 'Co', 'Private use', 'Cs', 'Surrogate', 'Ll', 'Lower case letter', 'Lm', 'Modifier letter', 'Lo', 'Other letter', 'Lt', 'Title case letter', 'Lu', 'Upper case letter', 'Mc', 'Spacing mark', 'Me', 'Enclosing mark', 'Mn', 'Non-spacing mark', 'Nd', 'Decimal number', 'Nl', 'Letter number', 'No', 'Other number', 'Pc', 'Connector punctuation', 'Pd', 'Dash punctuation', 'Pe', 'Close punctuation', 'Pf', 'Final punctuation', 'Pi', 'Initial punctuation', 'Po', 'Other punctuation', 'Ps', 'Open punctuation', 'Sc', 'Currency symbol', 'Sk', 'Modifier symbol', 'Sm', 'Mathematical symbol', 'So', 'Other symbol', 'Zl', 'Line separator', 'Zp', 'Paragraph separator', 'Zs', 'Space separator' ] # The Extended_Pictographic property is not found in the file where all the # others are (GraphemeBreakProperty.txt). It comes from the emoji-data.txt # file, but we list it here so that the name has the correct index value. break_properties = [ 'CR', ' 0', 'LF', ' 1', 'Control', ' 2', 'Extend', ' 3', 'Prepend', ' 4', 'SpacingMark', ' 5', 'L', ' 6 Hangul syllable type L', 'V', ' 7 Hangul syllable type V', 'T', ' 8 Hangul syllable type T', 'LV', ' 9 Hangul syllable type LV', 'LVT', '10 Hangul syllable type LVT', 'Regional_Indicator', '11', 'Other', '12', 'ZWJ', '13', 'Extended_Pictographic', '14' ] # List of files from which the names of Boolean properties are obtained, along # with a list of regex patterns for properties to be ignored, and a list of # extra pattern names to add. bool_propsfiles = ['PropList.txt', 'DerivedCoreProperties.txt', 'emoji-data.txt'] bool_propsignore = [r'^Other_', r'^Hyphen$'] bool_propsextras = ['ASCII', 'Bidi_Mirrored'] # --------------------------------------------------------------------------- # GET BOOLEAN PROPERTY NAMES # --------------------------------------------------------------------------- # Get a list of Boolean property names from a number of files. def getbpropslist(): bplist = [] bplast = "" for filename in bool_propsfiles: try: file = open('Unicode.tables/' + filename, 'r') except IOError: print(f"** Couldn't open {'Unicode.tables/' + filename}\n") sys.exit(1) for line in file: line = re.sub(r'#.*', '', line) data = list(map(str.strip, line.split(';'))) if len(data) <= 1 or data[1] == bplast: continue bplast = data[1] for pat in bool_propsignore: if re.match(pat, bplast) != None: break else: bplist.append(bplast) file.close() bplist.extend(bool_propsextras) bplist.sort() return bplist bool_properties = getbpropslist() bool_props_list_item_size = (len(bool_properties) + 31) // 32 # --------------------------------------------------------------------------- # COLLECTING PROPERTY NAMES AND ALIASES # --------------------------------------------------------------------------- script_names = ['Unknown'] abbreviations = {} def collect_property_names(): global script_names global abbreviations names_re = re.compile(r'^[0-9A-F]{4,6}(?:\.\.[0-9A-F]{4,6})? +; ([A-Za-z_]+) #') last_script_name = "" with open("Unicode.tables/Scripts.txt") as f: for line in f: match_obj = names_re.match(line) if match_obj == None or match_obj.group(1) == last_script_name: continue last_script_name = match_obj.group(1) script_names.append(last_script_name) # Sometimes there is comment in the line # so splitting around semicolon is not enough value_alias_re = re.compile(r' *([A-Za-z_]+) *; *([A-Za-z_]+) *; *([A-Za-z_]+)(?: *; *([A-Za-z_ ]+))?') with open("Unicode.tables/PropertyValueAliases.txt") as f: for line in f: match_obj = value_alias_re.match(line) if match_obj == None: continue if match_obj.group(1) == "sc": if match_obj.group(2) == match_obj.group(3): abbreviations[match_obj.group(3)] = () elif match_obj.group(4) == None: abbreviations[match_obj.group(3)] = (match_obj.group(2),) else: abbreviations[match_obj.group(3)] = (match_obj.group(2), match_obj.group(4)) # We can also collect Boolean property abbreviations into the same dictionary bin_alias_re = re.compile(r' *([A-Za-z_]+) *; *([A-Za-z_]+)(?: *; *([A-Za-z_]+))?') with open("Unicode.tables/PropertyAliases.txt") as f: for line in f: match_obj = bin_alias_re.match(line) if match_obj == None: continue if match_obj.group(2) in bool_properties: if match_obj.group(3) == None: abbreviations[match_obj.group(2)] = (match_obj.group(1),) else: abbreviations[match_obj.group(2)] = (match_obj.group(1), match_obj.group(3)) collect_property_names() # --------------------------------------------------------------------------- # REORDERING SCRIPT NAMES # --------------------------------------------------------------------------- script_abbrevs = [] def reorder_scripts(): global script_names global script_abbrevs global abbreviations for name in script_names: abbrevs = abbreviations[name] script_abbrevs.append(name if len(abbrevs) == 0 else abbrevs[0]) extended_script_abbrevs = set() with open("Unicode.tables/ScriptExtensions.txt") as f: names_re = re.compile(r'^[0-9A-F]{4,6}(?:\.\.[0-9A-F]{4,6})? +; ([A-Za-z_ ]+) #') for line in f: match_obj = names_re.match(line) if match_obj == None: continue for name in match_obj.group(1).split(" "): extended_script_abbrevs.add(name) new_script_names = [] new_script_abbrevs = [] for idx, abbrev in enumerate(script_abbrevs): if abbrev in extended_script_abbrevs: new_script_names.append(script_names[idx]) new_script_abbrevs.append(abbrev) for idx, abbrev in enumerate(script_abbrevs): if abbrev not in extended_script_abbrevs: new_script_names.append(script_names[idx]) new_script_abbrevs.append(abbrev) script_names = new_script_names script_abbrevs = new_script_abbrevs reorder_scripts() script_list_item_size = (script_names.index('Unknown') + 31) // 32 # --------------------------------------------------------------------------- # DERIVED LISTS # --------------------------------------------------------------------------- # Create general character property names from the first letters of the # particular categories. gcn_set = set(category_names[i][0] for i in range(0, len(category_names), 2)) general_category_names = list(gcn_set) general_category_names.sort() # --------------------------------------------------------------------------- # FUNCTIONS # --------------------------------------------------------------------------- import sys # Open an output file, using the command's argument or a default. Write common # preliminary header information. def open_output(default): if len(sys.argv) > 2: print('** Too many arguments: just give a file name') sys.exit(1) if len(sys.argv) == 2: output_name = sys.argv[1] else: output_name = default try: file = open(output_name, "w") except IOError: print ("** Couldn't open %s" % output_name) sys.exit(1) script_name = sys.argv[0] i = script_name.rfind('/') if i >= 0: script_name = script_name[i+1:] file.write("""\ /************************************************* * Perl-Compatible Regular Expressions * *************************************************/ /* PCRE is a library of functions to support regular expressions whose syntax and semantics are as close as possible to those of the Perl 5 language. Written by Philip Hazel Original API code Copyright (c) 1997-2012 University of Cambridge New API code Copyright (c) 2016-2022 University of Cambridge This module is auto-generated from Unicode data files. DO NOT EDIT MANUALLY! """) file.write("Instead, modify the maint/%s script and run it to generate\n" "a new version of this code.\n\n" % script_name) file.write("""\ ----------------------------------------------------------------------------- Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the University of Cambridge nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------------------------------------------------------------- */ \n""") return file # End of UcpCommon.py