blob: aadfa38149a7ba7dcd157dc138da3b063b5f96ca [file] [log] [blame]
#! /usr/bin/perl
#
# Copyright (c) 2001-2009, PostgreSQL Global Development Group
#
# $PostgreSQL: pgsql/src/backend/utils/mb/Unicode/UCS_to_most.pl,v 1.8 2009/03/18 16:17:28 heikki Exp $
#
# Generate UTF-8 <--> character code conversion tables from
# map files provided by Unicode organization.
# Unfortunately it is prohibited by the organization
# to distribute the map files. So if you try to use this script,
# you have to obtain the map files from the organization's ftp site.
# ftp://www.unicode.org/Public/MAPPINGS/
# We assume the file include three tab-separated columns:
# source character set code in hex
# UCS-2 code in hex
# # and Unicode name (not used in this script)
require "ucs2utf.pl";
%filename = (
'WIN866' => 'CP866.TXT',
'WIN874' => 'CP874.TXT',
'WIN1250' => 'CP1250.TXT',
'WIN1251' => 'CP1251.TXT',
'WIN1252' => 'CP1252.TXT',
'WIN1253' => 'CP1253.TXT',
'WIN1254' => 'CP1254.TXT',
'WIN1255' => 'CP1255.TXT',
'WIN1256' => 'CP1256.TXT',
'WIN1257' => 'CP1257.TXT',
'WIN1258' => 'CP1258.TXT',
'ISO8859_2' => '8859-2.TXT',
'ISO8859_3' => '8859-3.TXT',
'ISO8859_4' => '8859-4.TXT',
'ISO8859_5' => '8859-5.TXT',
'ISO8859_6' => '8859-6.TXT',
'ISO8859_7' => '8859-7.TXT',
'ISO8859_8' => '8859-8.TXT',
'ISO8859_9' => '8859-9.TXT',
'ISO8859_10' => '8859-10.TXT',
'ISO8859_13' => '8859-13.TXT',
'ISO8859_14' => '8859-14.TXT',
'ISO8859_15' => '8859-15.TXT',
'ISO8859_16' => '8859-16.TXT',
'KOI8R' => 'KOI8-R.TXT',
'KOI8U' => 'KOI8-U.TXT',
'GBK' => 'CP936.TXT',
'UHC' => 'CP949.TXT',
'JOHAB' => 'JOHAB.TXT',
);
@charsets = keys(filename);
@charsets = @ARGV if scalar(@ARGV);
foreach $charset (@charsets) {
#
# first, generate UTF8-> charset table
#
$in_file = $filename{$charset};
open( FILE, $in_file ) || die( "cannot open $in_file" );
reset 'array';
while( <FILE> ){
chop;
if( /^#/ ){
next;
}
( $c, $u, $rest ) = split;
$ucs = hex($u);
$code = hex($c);
if( $code >= 0x80 && $ucs >= 0x0080){
$utf = &ucs2utf($ucs);
if( $array{ $utf } ne "" ){
printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs;
next;
}
$count++;
$array{ $utf } = $code;
}
}
close( FILE );
$file = lc("utf8_to_${charset}.map");
open( FILE, "> $file" ) || die( "cannot open $file" );
print FILE "static pg_utf_to_local ULmap${charset}[ $count ] = {\n";
for $index ( sort {$a <=> $b} keys( %array ) ){
$code = $array{ $index };
$count--;
if( $count == 0 ){
printf FILE " {0x%04x, 0x%04x}\n", $index, $code;
} else {
printf FILE " {0x%04x, 0x%04x},\n", $index, $code;
}
}
print FILE "};\n";
close(FILE);
#
# then generate character set code ->UTF8 table
#
open( FILE, $in_file ) || die( "cannot open $in_file" );
reset 'array';
while( <FILE> ){
chop;
if( /^#/ ){
next;
}
( $c, $u, $rest ) = split;
$ucs = hex($u);
$code = hex($c);
if($code >= 0x80 && $ucs >= 0x0080){
$utf = &ucs2utf($ucs);
if( $array{ $code } ne "" ){
printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs;
next;
}
$count++;
$array{ $code } = $utf;
}
}
close( FILE );
$file = lc("${charset}_to_utf8.map");
open( FILE, "> $file" ) || die( "cannot open $file" );
print FILE "static pg_local_to_utf LUmap${charset}[ $count ] = {\n";
for $index ( sort {$a <=> $b} keys( %array ) ){
$utf = $array{ $index };
$count--;
if( $count == 0 ){
printf FILE " {0x%04x, 0x%04x}\n", $index, $utf;
} else {
printf FILE " {0x%04x, 0x%04x},\n", $index, $utf;
}
}
print FILE "};\n";
close(FILE);
}