tools - zygaena/hammerhead

root / base / usr / src / data / locale / tools
tools Plain Text 502 lines 15.3 KB
#
# This file and its contents are supplied under the terms of the
# Common Development and Distribution License ("CDDL"), version 1.0.
# You may only use this file in accordance with the terms of version
# 1.0 of the CDDL.
#
# A full copy of the text of the CDDL should have accompanied this
# source.  A copy of the CDDL is also available via the Internet at
# http://www.illumos.org/license/CDDL.
#

#
# Copyright 2017 Nexenta Systems, Inc.
#

include		$(SRC)/data/Makefile.data
include		$(SRC)/data/locale/Makefile.com

CMFILES=	$(_CHARMAPS:%=build/%.cm)

GB18030SRCS=	$(GB18030_LOCALES:%=build/%.GB18030.src)
ISO1SRCS=	$(ISO8859_1_LOCALES:%=build/%.ISO8859-1.src)
ISO2SRCS=	$(ISO8859_2_LOCALES:%=build/%.ISO8859-2.src)
ISO5SRCS=	$(ISO8859_5_LOCALES:%=build/%.ISO8859-5.src)
ISO7SRCS=	$(ISO8859_7_LOCALES:%=build/%.ISO8859-7.src)
ISO9SRCS=	$(ISO8859_9_LOCALES:%=build/%.ISO8859-9.src)
ISO11SRCS=	$(_ISO8859_11_LOCALES:%=build/%.ISO8859-11.src)
ISO13SRCS=	$(ISO8859_13_LOCALES:%=build/%.ISO8859-13.src)
ISO15SRCS=	$(ISO8859_15_LOCALES:%=build/%.ISO8859-15.src)
KOI8RSRCS=	$(KOI8_R_LOCALES:%=build/%.KOI8-R.src)
UTF8SRCS=	$(_UTF_8_LOCALES:%=build/%.UTF-8.src)

.PARALLEL:

all:		$(CMFILES) $(GB18030SRCS) $(ISO1SRCS) $(ISO2SRCS) $(ISO5SRCS) \
		$(ISO7SRCS) $(ISO9SRCS) $(ISO13SRCS) $(ISO15SRCS) $(KOI8RSRCS) \
		$(UTF8SRCS)

$(CMFILES):	build posix.jar
		java -jar posix.jar org.unicode.cldr.posix.GenerateCharmap -d build -c $(@F:%.cm=%)

$(GB18030SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.GB18030.src=%) -c GB18030
$(ISO1SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.ISO8859-1.src=%) -c ISO8859-1
$(ISO2SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.ISO8859-2.src=%) -c ISO8859-2
$(ISO5SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.ISO8859-5.src=%) -c ISO8859-5
$(ISO7SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.ISO8859-7.src=%) -c ISO8859-7
$(ISO9SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.ISO8859-9.src=%) -c ISO8859-9
$(ISO13SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.ISO8859-13.src=%) -c ISO8859-13
$(ISO15SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.ISO8859-15.src=%) -c ISO8859-15
$(KOI8RSRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.KOI8-R.src=%) -c KOI8-R
$(UTF8SRCS):	build posix.jar
		java -DCLDR_DIR=cldr -jar posix.jar org.unicode.cldr.posix.GeneratePOSIX -d build -m $(@F:%.UTF-8.src=%) -c UTF-8

build:
		$(INS.dir)

posix.jar: posix.xml
		ant -f posix.xml
#!/bin/python
"""

This file and its contents are supplied under the terms of the
Common Development and Distribution License ("CDDL"), version 1.0.
You may only use this file in accordance with the terms of version
1.0 of the CDDL.

A full copy of the text of the CDDL should have accompanied this
source.  A copy of the CDDL is also available via the Internet at
http://www.illumos.org/license/CDDL.

Copyright 2013 DEY Storage Systems, Inc.

Scratch script to produce the widths.cm content from the widths text
files.  It converts numeric unicode to symbolic forms.
"""

# Copyright 2018 OmniOS Community Edition (OmniOSce) Association.

from __future__ import print_function

SYMBOLS = {}


def u8_str(val):
    """
    Convert a numeric value to a string representing the UTF-8 encoding
    of the numeric value, which should be a valid Unicode code point.
    """
    u8str = unichr(val).encode('utf-8')
    idx = 0
    out = ""
    while idx < len(u8str):
        out += "\\x%X" % ord(u8str[idx])
        idx += 1
    return out


def load_utf8():
    """
    This function loads the UTF-8 character map file, loading the symbols
    and the numeric values.  The result goes into the global SYMBOLS array.
    """
    lines = open("UTF-8.cm").readlines()
    for line in lines:
        items = line.split()
        if (len(items) != 2) or items[0].startswith("#"):
            continue
        (sym, val) = (items[0], items[1])
        SYMBOLS[val] = sym


def do_width_file(width, filename):
    """
    This function takes a file pairs of unicode values (hex), each of
    which is a range of unicode values, that all have the given width.
    """
    for line in open(filename).readlines():
        if line.startswith("#"):
            continue
        vals = line.split()
        while len(vals) > 1:
            start = int(vals[0], 16)
            end = int(vals[1], 16)
            val = start
            while val <= end:
                key = u8_str(val)
                val += 1
                sym = SYMBOLS.get(key, None)
                if sym == None:
                    continue
                print("%s\t%d" % (sym, width))
            vals = vals[2:]


if __name__ == "__main__":
    print("WIDTH")
    load_utf8()
    do_width_file(0, "widths-0.txt")
    do_width_file(2, "widths-2.txt")
    print("END WIDTH")
<!--
	This is a stripped down version of tools/java/build.xml
	from CLDR.  Generally, CLDR tools require Java 1.8, but
	POSIX conversion tool that we need can be run using older
	versions.
-->

<project name="cldr" default="posix" basedir=".">
	<target name="init">
		<tstamp />
		<property name="src.dir" value="cldr/tools/java" />
		<property name="build.dir" value="cldr/tools/java/classes" />
		<property name="libs.dir" value="cldr/tools/java/libs" />
		<property name="jar.file" value="posix.jar" />
		<property environment="env" />
		<property file="cldr/tools/java/cldr-tools.properties" />
		<path id="project.class.path">
			<pathelement path="${java.class.path}/" />
			<pathelement path="${build.dir}" />
			<fileset dir="${libs.dir}" includes="*.jar" />
		</path>
	</target>

	<target name="posix" depends="init" description="posix conversion tool">
		<property name="CLDR_DIR" value="cldr"/>
		<property name="CLDR_TOOLS" value="cldr/tools/java"/>
		<property file="${CLDR_DIR}/cldr-tools.properties" />
		<path id="build.classpath">
			<pathelement path="${build.dir}" />
			<pathelement path="${libs.dir}" />
		</path>
		<property name="jvm_options"
			value="-DCLDR_DIR=cldr -DDEBUG=false -Dverbose=false" />
		<echo message="java home: ${java.home}" />
		<echo message="java version: ${java.version}" />
		<echo message="ant java version: ${ant.java.version}" />
		<echo message="${ant.version}" />

		<mkdir dir="${build.dir}" />
		<javac includeantruntime="false"
			includes="org/unicode/cldr/posix/GenerateCharmap.java
			org/unicode/cldr/posix/GeneratePOSIX.java
			org/unicode/cldr/tool/Main.java"
			srcdir="${src.dir}" destdir="${build.dir}"
			classpathref="project.class.path"
			source="1.7" target="1.7" debug="on"
			deprecation="off" encoding="UTF-8" />
		<mkdir dir="${build.dir}/org/unicode/cldr/util/data" />
		<copy todir="${build.dir}/org/unicode/cldr/util/data">
			<fileset dir="${src.dir}/org/unicode/cldr/util/data" />
		</copy>
		<jar jarfile="${jar.file}" compress="true"
			includes="org/unicode/cldr/draft/**/*
			org/unicode/cldr/icu/**/*
			org/unicode/cldr/posix/**/*
			org/unicode/cldr/test/**/*
			org/unicode/cldr/tool/**/*
			org/unicode/cldr/util/**/*
			com/ibm/icu/**/*"
			basedir="${build.dir}">
			<manifest>
				<attribute name="Main-Class"
					value="org.unicode.cldr.tool.Main" />
				<attribute name="Class-Path"
					value="${libs.dir}/${cldr.libs.icu4j}
					${libs.dir}/${cldr.libs.utilities}
					${libs.dir}/${cldr.libs.xerces}
					${libs.dir}/${cldr.libs.guava}
					${libs.dir}/${cldr.libs.gson}
					${cldr.libs.icu4j}
					${cldr.libs.utilities}
					${cldr.libs.xerces}
					${cldr.libs.gson}" />
			</manifest>
		</jar>
	</target>

	<target name="clean" depends="init" description="remove build targets">
		<delete dir="${build.dir}" />
		<delete file="${jar.file}" />
	</target>
</project>
#!/usr/bin/perl -wC

#
# Copyright 2009 Edwin Groothuis <edwin@FreeBSD.org>
# Copyright 2015 John Marino <draco@marino.st>
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#

use strict;
use Getopt::Long;

if ($#ARGV != 0) {
	print "Usage: $0 --unidata=</path/to/UnicodeData.txt>\n";
	exit(1);
}

my $UNIDATA = undef;

my $result = GetOptions (
		"unidata=s"	=> \$UNIDATA
	    );

my %utf8map = ();
my $outfilename = "data/common.UTF-8.src";

get_utf8map("data/UTF-8.cm");
generate_header ();
parse_unidata ("$UNIDATA");
generate_footer ();

############################

sub get_utf8map {
	my $file = shift;

	open(FIN, $file);
	my @lines = <FIN>;
	close(FIN);
	chomp(@lines);

	my $incharmap = 0;
	foreach my $l (@lines) {
		$l =~ s/\r//;
		next if ($l =~ /^\#/);
		next if ($l eq "");

		if ($l eq "CHARMAP") {
			$incharmap = 1;
			next;
		}

		next if (!$incharmap);
		last if ($l eq "END CHARMAP");

		$l =~ /^(<[^\s]+>)\s+(.*)/;
		my $k = $2;
		my $v = $1;
		$k =~ s/\\x//g;		# UTF-8 char code
		$utf8map{$k} = $v;
	}
}

sub generate_header {
	open(FOUT, ">", "$outfilename")
		or die ("can't write to $outfilename\n");
	print FOUT "LC_CTYPE\n\n";
}

sub generate_footer {
	print FOUT "\nEND LC_CTYPE\n";
	close (FOUT);
}

sub wctomb {
	my $wc = hex(shift);
	my $lead;
	my $len;
	my $ret = "";
	my $i;

	if (($wc & ~0x7f) == 0) {
		return sprintf "%02X", $wc;
	} elsif (($wc & ~0x7ff) == 0) {
		$lead = 0xc0;
		$len = 2;
	} elsif (($wc & ~0xffff) == 0) {
		$lead = 0xe0;
		$len = 3;
	} elsif ($wc >= 0 && $wc <= 0x10ffff) {
		$lead = 0xf0;
		$len = 4;
	}

	for ($i = $len - 1; $i > 0; $i--) {
		$ret = (sprintf "%02X", ($wc & 0x3f) | 0x80) . $ret;
		$wc >>= 6;
	}
	$ret = (sprintf "%02X", ($wc & 0xff) | $lead) . $ret;

	return $ret;
}

sub parse_unidata {
	my $file = shift;
	my %data = ();

	open(FIN, $file);
	my @lines = <FIN>;
	close(FIN);
	chomp(@lines);

	foreach my $l (@lines) {
		my @d = split(/;/, $l, -1);
		my $mb = wctomb($d[0]);
		my $cat;

		# XXX There are code points present in UnicodeData.txt
		# and missing from UTF-8.cm
		next if !defined $utf8map{$mb};

		# Define the category
		if ($d[2] =~ /^Lu/) {
			$cat = "upper";
		} elsif ($d[2] =~ /^Ll/) {
			$cat = "lower";
		} elsif ($d[2] =~ /^Nd/) {
			$cat = "digit";
		} elsif ($d[2] =~ /^L/) {
			$cat = "alpha";
		} elsif ($d[2] =~ /^P/) {
			$cat = "punct";
		} elsif ($d[2] =~ /^Co/ || $d[2] =~ /^M/ || $d[2] =~ /^N/ ||
		    $d[2] =~ /^S/) {
			$cat = "graph";
		} elsif ($d[2] =~ /^C/) {
			$cat = "cntrl";
		} elsif ($d[2] =~ /^Z/) {
			$cat = "space";
		}
		$data{$cat}{$mb}{'wc'} = $d[0];

		# Check if it's a start or end of range
		if ($d[1] =~ /First>$/) {
			$data{$cat}{$mb}{'start'} = 1;
		} elsif ($d[1] =~ /Last>$/) {
			$data{$cat}{$mb}{'end'} = 1;
		}

		# Check if there's upper/lower mapping
		if ($d[12] ne "") {
			$data{'toupper'}{$mb} = wctomb($d[12]);
		} elsif ($d[13] ne "") {
			$data{'tolower'}{$mb} = wctomb($d[13]);
		}
	}

	my $first;
	my $inrange = 0;

	# Now write out the categories
	foreach my $cat (sort keys (%data)) {
		print FOUT "$cat\t";
		$first = 1;
	foreach my $mb (sort keys (%{$data{$cat}})) {
		if ($first == 1) {
			$first = 0;
		} elsif ($inrange == 1) {
			# Safety belt
			die "broken range end wc=$data{$cat}{$mb}{'wc'}"
			    if !defined $data{$cat}{$mb}{'end'};
			print FOUT ";...;";
			$inrange = 0;
		} else {
			print FOUT ";/\n\t";
		}

		if ($cat eq "tolower" || $cat eq "toupper") {
			print FOUT "($utf8map{$mb},$utf8map{$data{$cat}{$mb}})";
		} else {
			if (defined($data{$cat}{$mb}{'start'})) {
				$inrange = 1;
			}
			print FOUT "$utf8map{$mb}";
		}
	}
		print FOUT "\n";
	}
}
#
# This file is a list of pairs of ranges of Unicode values. Characters
# within these ranges (inclusive) have screen width 0.
#
     0x0300 0x036F   0x0483 0x0486   0x0488 0x0489
     0x0591 0x05BD   0x05BF 0x05BF   0x05C1 0x05C2
     0x05C4 0x05C5   0x05C7 0x05C7   0x0600 0x0603
     0x0610 0x0615   0x064B 0x065E   0x0670 0x0670
     0x06D6 0x06E4   0x06E7 0x06E8   0x06EA 0x06ED
     0x070F 0x070F   0x0711 0x0711   0x0730 0x074A
     0x07A6 0x07B0   0x07EB 0x07F3   0x0901 0x0902
     0x093C 0x093C   0x0941 0x0948   0x094D 0x094D
     0x0951 0x0954   0x0962 0x0963   0x0981 0x0981
     0x09BC 0x09BC   0x09C1 0x09C4   0x09CD 0x09CD
     0x09E2 0x09E3   0x0A01 0x0A02   0x0A3C 0x0A3C
     0x0A41 0x0A42   0x0A47 0x0A48   0x0A4B 0x0A4D
     0x0A70 0x0A71   0x0A81 0x0A82   0x0ABC 0x0ABC
     0x0AC1 0x0AC5   0x0AC7 0x0AC8   0x0ACD 0x0ACD
     0x0AE2 0x0AE3   0x0B01 0x0B01   0x0B3C 0x0B3C
     0x0B3F 0x0B3F   0x0B41 0x0B43   0x0B4D 0x0B4D
     0x0B56 0x0B56   0x0B82 0x0B82   0x0BC0 0x0BC0
     0x0BCD 0x0BCD   0x0C3E 0x0C40   0x0C46 0x0C48
     0x0C4A 0x0C4D   0x0C55 0x0C56   0x0CBC 0x0CBC
     0x0CBF 0x0CBF   0x0CC6 0x0CC6   0x0CCC 0x0CCD
     0x0CE2 0x0CE3   0x0D41 0x0D43   0x0D4D 0x0D4D
     0x0DCA 0x0DCA   0x0DD2 0x0DD4   0x0DD6 0x0DD6
     0x0E31 0x0E31   0x0E34 0x0E3A   0x0E47 0x0E4E
     0x0EB1 0x0EB1   0x0EB4 0x0EB9   0x0EBB 0x0EBC
     0x0EC8 0x0ECD   0x0F18 0x0F19   0x0F35 0x0F35
     0x0F37 0x0F37   0x0F39 0x0F39   0x0F71 0x0F7E
     0x0F80 0x0F84   0x0F86 0x0F87   0x0F90 0x0F97
     0x0F99 0x0FBC   0x0FC6 0x0FC6   0x102D 0x1030
     0x1032 0x1032   0x1036 0x1037   0x1039 0x1039
     0x1058 0x1059   0x1160 0x11FF   0x135F 0x135F
     0x1712 0x1714   0x1732 0x1734   0x1752 0x1753
     0x1772 0x1773   0x17B4 0x17B5   0x17B7 0x17BD
     0x17C6 0x17C6   0x17C9 0x17D3   0x17DD 0x17DD
     0x180B 0x180D   0x18A9 0x18A9   0x1920 0x1922
     0x1927 0x1928   0x1932 0x1932   0x1939 0x193B
     0x1A17 0x1A18   0x1B00 0x1B03   0x1B34 0x1B34
     0x1B36 0x1B3A   0x1B3C 0x1B3C   0x1B42 0x1B42
     0x1B6B 0x1B73   0x1DC0 0x1DCA   0x1DFE 0x1DFF
     0x200B 0x200F   0x202A 0x202E   0x2060 0x2063
     0x206A 0x206F   0x20D0 0x20EF   0x302A 0x302F
     0x3099 0x309A   0xA806 0xA806   0xA80B 0xA80B
     0xA825 0xA826   0xFB1E 0xFB1E   0xFE00 0xFE0F
     0xFE20 0xFE23   0xFEFF 0xFEFF   0xFFF9 0xFFFB
#
# This file is a list of pairs of ranges of Unicode values. Characters
# within these ranges (inclusive) have screen width 2.
#
0x1100 0x115f
0x2329 0x2329
0x232a 0x232a
0x2e80 0x303e
0x3040 0xa4cf
0xac00 0xd7a3
0xf900 0xfaff
0xfe10 0xfe19
0xfe30 0xfe6f
0xff00 0xff60
0xffe0 0xffe6