util/codecvt.cpp

/***************************************************************************
 *
 * codecvt.cpp
 *
 * $Id$
 *
 ***************************************************************************
 *
 * Licensed to the Apache Software  Foundation (ASF) under one or more
 * contributor  license agreements.  See  the NOTICE  file distributed
 * with  this  work  for  additional information  regarding  copyright
 * ownership.   The ASF  licenses this  file to  you under  the Apache
 * License, Version  2.0 (the  "License"); you may  not use  this file
 * except in  compliance with the License.   You may obtain  a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the  License is distributed on an  "AS IS" BASIS,
 * WITHOUT  WARRANTIES OR CONDITIONS  OF ANY  KIND, either  express or
 * implied.   See  the License  for  the  specific language  governing
 * permissions and limitations under the License.
 *
 * Copyright 2001-2006 Rogue Wave Software.
 * 
 **************************************************************************/

#include "diagnostic.h"   // for issue_diag()
#include "def.h"          // for Def

#include "path.h"         // for get_pathname()
#include "scanner.h"      // for scanner

#include <cassert>        // for assert()
#include <climits>        // for UCHAR_MAX
#include <cstring>        // for memset()
#include <fstream>        // for ifstream, ofstream


typedef std::map<std::string, wchar_t>::const_iterator n_cmap_citer2;


std::size_t Def::
gen_mbchar_tables (codecvt_offsets_map_t           &tab,
                   std::map<std::string, unsigned> &off_map,
                   const std::string               &charp /* = "" */,
                   unsigned                         tabno /* = 0 */)
{
    // upon the first call (but not during subsequent recursive calls)
    // generate a set of multibyte prefixes from the set of all known
    // multibyte characters
    static unsigned               ntabs   = 0;
    static std::set<std::string>* pfx_set = 0;

    const n_cmap_citer2 mb_map_end = charmap_.get_mb_cmap ().end ();

    if (0 == pfx_set) {
        pfx_set = new std::set<std::string>;

        // iterate over the range of valid multibyte characters
        // obtained from the charmap and generate a complete
        // subset of non-empty multibyte prefixes from each
        unsigned off = 0;

        const n_cmap_citer2 mb_map_begin = charmap_.get_mb_cmap ().begin ();

        for (n_cmap_citer2 it = mb_map_begin; it != mb_map_end; ++it, ++off) {

            // insert the ordinal number of each multibyte character
            // into a map for fast lookup later
            off_map.insert (std::make_pair (it->first, off));

            // generate non-empty prefixes up to one byte less
            // in length than the complete multibyte character
            for (std::string prefix = it->first; 1 < prefix.size (); ) {
                prefix = prefix.substr (0, prefix.size () - 1);
                pfx_set->insert (prefix);
            }
        }
    }

    // number of valid characters inserted into the tables
    std::size_t nchars = 0;

    // an array of offsets to the multibyte character or to the next
    // array containing such offsets (defined recursively for up to
    // MB_CUR_MAX levels of nesting)
    codecvt_offset_tab_t* const offsets = new codecvt_offset_tab_t;

    std::string mb_char (charp + '\0');

    for (unsigned i = 0; i <= UCHAR_MAX; ++i) {

        unsigned char cur_char = (unsigned char)i;

        mb_char [mb_char.size () - 1] = char (cur_char);

        if (mb_map_end == charmap_.get_mb_cmap ().find (mb_char)) {
            // mb_char is not a complete, valid multibyte character
            // check to see if it's a prefix of one

            if (pfx_set->find (mb_char) == pfx_set->end ()) {
                // mb_char is not a prefix of a valid multibyte
                // character, mark it invalide
                offsets->off [cur_char] = UINT_MAX;
            }
            else {
                // mb_char is a prefix of a valid multibyte character,
                // set the MSB to denote that it "continues" in the
                // table at the next higher offset
                offsets->off [cur_char] = ++ntabs | 0x80000000;

                // generate that table
                nchars += gen_mbchar_tables (tab, off_map, mb_char, ntabs);
            }
        }
        else {
            // mb_char is a complete, valid miltibyte character
            // insert its ordinal number (offset) into the array
            offsets->off [cur_char] = off_map.find (mb_char)->second;
            ++nchars;
        }
    }

    // insert the completely populated table into the map
    tab.insert (std::make_pair (tabno, offsets));

    if (0 == ntabs) {
        // clean up on return from the topmost (non-recursive) call
        delete pfx_set;
        pfx_set = 0;
    }

    return nchars;
}


std::size_t Def::
gen_wchar_tables (codecvt_offsets_map_t &tab,
                  const std::string     &charp /* = "" */, 
                  unsigned int           tabno /* = 0 */)
{
    // upon the first call (but not during subsequent recursive calls)
    // generate a set of multibyte prefixes from the set of all known
    // multibyte characters
    static unsigned                            ntabs   = 0;
    static std::set<std::string>              *pfx_set = 0;
    static std::map<std::string, unsigned>    *off_map = 0;
    static std::map<std::string, std::string> *utf_map = 0;

    if (0 == utf_map) {
        pfx_set = new std::set<std::string>;
        off_map = new std::map<std::string, unsigned>;
        utf_map = new std::map<std::string, std::string>;

        const n_cmap_citer2 first = charmap_.get_mb_cmap ().begin ();
        const n_cmap_citer2 last  = charmap_.get_mb_cmap ().end ();

        unsigned off = 0;

        for (n_cmap_citer2 it = first; it != last; ++it) {

            off_map->insert (std::make_pair (it->first, off));

            off += it->first.size () + 1;

            std::string utf = utf8_encode (it->second);

            utf_map->insert (std::make_pair (utf, it->first));

            while (1 < utf.size ()) {
                utf = utf.substr (0, utf.size () - 1);
                pfx_set->insert (utf);
            }
        }
    }

    codecvt_offset_tab_t* const offsets = new codecvt_offset_tab_t;

    // number of valid characters inserted into the tables
    std::size_t nchars = 0;

    std::string mb_char (charp + '\0');

    for (unsigned i = 0; i <= UCHAR_MAX; ++i) {

        unsigned char cur_char = (unsigned char)i;

        mb_char [mb_char.size () - 1] = char (cur_char);

        const wchar_utf8_iter it = utf_map->find (mb_char);
        if (it == utf_map->end ()) {
            if (pfx_set->find (mb_char) == pfx_set->end ()) {
                offsets->off [cur_char] = UINT_MAX;
            }
            else {
                offsets->off [cur_char] = ++ntabs | 0x80000000;

                nchars += gen_wchar_tables (tab, mb_char, ntabs);
            }
        }
        else {
            offsets->off [cur_char] = off_map->find (it->second)->second;

            ++nchars;
        }
    }

    tab.insert (std::make_pair (tabno, offsets));

    if (0 == ntabs) {
        // clean up
        delete pfx_set;
        delete utf_map;

        pfx_set = 0;
        utf_map = 0;
    }

    return nchars;
}


std::size_t Def::
gen_utf8_tables (codecvt_offsets_map_t           &tab,
                 std::map<std::string, unsigned> &off_map,
                 const std::string               &charp /* = "" */,
                 unsigned                         tabno /* = 0 */)
{
    static unsigned                        ntabs   = 0;
    static std::set<std::string>          *pfx_set = 0;
    static std::map<std::string, wchar_t> *utf_map = 0;

    if (0 == pfx_set) {
        pfx_set = new std::set<std::string>;

        const ucs4_cmap_iter first = charmap_.get_ucs4_cmap ().begin ();
        const ucs4_cmap_iter last  = charmap_.get_ucs4_cmap ().end ();

        for (ucs4_cmap_iter it = first; it != last; ++it) {

            for (std::string prefix = utf8_encode (it->second);
                 1 < prefix.size (); ) {
                prefix = prefix.substr (0, prefix.size () - 1);
                pfx_set->insert (prefix);
            }
        }
    }

    // the set of complete utf8 strings in the current character map
    typedef std::map<std::string, wchar_t>::iterator utf8_map_iter;

    if (0 == utf_map) {
        utf_map = new std::map<std::string, wchar_t>;
    
        const ucs4_cmap_iter first = charmap_.get_ucs4_cmap ().begin ();
        const ucs4_cmap_iter last  = charmap_.get_ucs4_cmap ().end ();

        for (ucs4_cmap_iter it = first; it != last; ++it) {
            const std::string utf = utf8_encode (it->second);
            utf_map->insert (std::make_pair (utf, it->second));
        }
    }

    codecvt_offset_tab_t* const offsets = new codecvt_offset_tab_t;

    // number of valid characters inserted into the tables
    std::size_t nchars = 0;

    std::string mb_char = charp + '\0';

    for (unsigned int i = 0; i <= UCHAR_MAX; ++i) {

        unsigned char cur_char = (unsigned char)i;

        mb_char [mb_char.size () - 1] = char (cur_char);

        const utf8_map_iter where = utf_map->find (mb_char);

        if (where == utf_map->end ()) {
            if (pfx_set->find (mb_char) == pfx_set->end ()) {
                offsets->off [cur_char] = UINT_MAX;
            }
            else {
                offsets->off [cur_char] = ++ntabs | 0x80000000;
                nchars += gen_utf8_tables (tab, off_map, mb_char, ntabs);
            }
        }
        else {
            // first get the symbolic name
            std::string str
                = charmap_.get_rucs4_cmap ().find (where->second)->second;

            // then get the internal encoding of the character
            const wchar_t int_enc = charmap_.get_w_cmap().find (str)->second;

            // then get the external encoding to use in a lookup in 
            // mb_char_off_map
            str = charmap_.get_rmb_cmap ().find (int_enc)->second;

            offsets->off [cur_char] = off_map.find (str)->second;

            ++nchars;
        }
    }

    tab.insert (std::make_pair (tabno, offsets));

    if (0 == ntabs) {
        // clean up
        delete pfx_set;
        delete utf_map;

        pfx_set = 0;
        utf_map = 0;
    }
    return nchars;
}


void Def::
gen_xlit_data ()
{
    // data offset points to the beginning of the data containing
    // the narrow strings character encodings
    unsigned int data_offset = 0;

    // traverse the map and construct the map of offsets
    xlit_map_t::const_iterator it = xlit_map_.begin ();
    for (; it != xlit_map_.end (); ++it) {
        // insert pair(wchar_t value, offset of first string in data block)
        xlit_data_offset_map_.insert (
            std::make_pair (it->first,data_offset));

        // advance the data_offset value to the next "first" string
        std::list<std::string>::const_iterator sit = 
            it->second.begin ();
        for (; sit != it->second.end (); ++sit) {
            data_offset += sit->size () + 1;
        }
        ++data_offset;
    }

    // create a new table (first), populate it with default values
    // and insert it in the tables map
    xlit_offset_table_t table0;
    unsigned int k;
    for (k = 0; k < UCHAR_MAX + 1; ++k)
        table0.offset_table [k] = UINT_MAX;

    // insert it into the map
    xlit_table_map_.insert (std::make_pair(0, table0));

    const xlit_map_t::const_iterator xlit_map_end = xlit_map_.end ();

    // traverse the map again and build the tables
    for (it = xlit_map_.begin (); it != xlit_map_end; ++it) {

        // encode the wchar_t value to UTF-8
        const std::string utf8_rep (utf8_encode (it->first));
        data_offset = xlit_data_offset_map_.find (it->first)->second;

        // traverse the utf8 representation string and create the 
        // necessary tables and populate the indexes
        unsigned int table_idx = 0;

        const std::string::const_iterator utf8_rep_end = utf8_rep.end ();
        std::string::const_iterator       string_it    = utf8_rep.begin ();

        for (; string_it != utf8_rep_end; ++string_it) {
            // get the table corresponding to the current index and locate
            // the value at that index
            const xlit_table_map_t::iterator res =
                xlit_table_map_.find (table_idx);

            assert (res != xlit_table_map_.end ());

            // offset in table
            unsigned char off_idx = (unsigned char)*string_it;

            // res is the iterator pointing to the correct table in the map
            // check the index and if not populated, create a new table
            if (res->second.offset_table [off_idx] == UINT_MAX) {

                // if this is the last position in the string, then
                // fill the table position with the offset of the string data
                if ((string_it + 1) == utf8_rep.end ()) {
                    xlit_data_offset_map_t::const_iterator data_it = 
                        xlit_data_offset_map_.find (it->first);
                    assert (data_it != xlit_data_offset_map_.end ());

                    // fill the table position with the found offset
                    res->second.offset_table [off_idx] = data_it->second;
                    continue;
                }

                // create a new table and append it to the map
                xlit_offset_table_t table;
                for (unsigned int i = 0; i < UCHAR_MAX + 1; ++i)
                    table.offset_table [i] = UINT_MAX;

                // insert it into the map
                unsigned int tmp = xlit_table_map_.size ();
                xlit_table_map_.insert (std::make_pair(tmp, table));
                
                // store its index at correct position in current table
                res->second.offset_table [off_idx] = tmp | 0x80000000;
                table_idx = tmp;
            } else {
                table_idx = 
                    res->second.offset_table [off_idx] & 0x7FFFFFFF;
            }
        }
    }
}


void Def::
write_codecvt (std::string dir_name)
{
    // if it has been already written
    if (codecvt_written_)
        return;

    // compose the directory name 
    ((dir_name += _RWSTD_PATH_SEP) += "..") += _RWSTD_PATH_SEP;
    dir_name += charmap_.get_code_set_name ();

    // check to see if the codecvt database already exists and
    // avoid recreating it if it does (as an optimization)
    if (std::ifstream (dir_name.c_str ())) {
        issue_diag (I_OPENWR, false, 0,
                    "%s exists, skipping\n", dir_name.c_str ());
        return;
    }

    //////////////////////////////////////////////////////////////////
    // generate multibyte conversion tables
    issue_diag (I_STAGE, false, 0, "generating multibyte tables\n");

    codecvt_offsets_map_t           mbchar_offs;
    std::map<std::string, unsigned> off_map;
    const std::size_t n_mbchars = gen_mbchar_tables (mbchar_offs, off_map);

    // generate wchar_t conversion tables
    issue_diag (I_STAGE, false, 0, "generating wchar_t tables\n");

    codecvt_offsets_map_t wchar_offs;
    const std::size_t n_wchars = gen_wchar_tables (wchar_offs);

    // generate UTF-8 conversion conversion tables
    issue_diag (I_STAGE, false, 0, "generating UTF-8 tables\n");

    codecvt_offsets_map_t uchar_offs;
    const std::size_t n_uchars = gen_utf8_tables (uchar_offs, off_map);

    // not needed beyond this point, clear it out
    off_map.clear ();
    
    // generate the transliteration tables and the transliteration data
    issue_diag (I_STAGE, false, 0, "generating transliteration tables\n");
    gen_xlit_data ();

    //////////////////////////////////////////////////////////////////
    // populate the codecvt structure before writing it out
    // in binary form to the file (the codecvt database)
    _RW::__rw_codecvt_t codecvt_out;
    std::memset (&codecvt_out, 0, sizeof codecvt_out);

    // calculate byte offsets within the structure
    codecvt_out.n_to_w_tab_off = 0;
    codecvt_out.w_to_n_tab_off = codecvt_out.n_to_w_tab_off
        + mbchar_offs.size () * (UCHAR_MAX + 1) * sizeof (unsigned);

    codecvt_out.utf8_to_ext_tab_off = codecvt_out.w_to_n_tab_off
        + wchar_offs.size () * (UCHAR_MAX + 1) * sizeof (unsigned);

    // insert the transliteration tables here
    codecvt_out.xliteration_off = codecvt_out.utf8_to_ext_tab_off
        + uchar_offs.size () * (UCHAR_MAX + 1) * sizeof (unsigned);

    codecvt_out.wchar_off = codecvt_out.xliteration_off
        + xlit_table_map_.size () * (UCHAR_MAX + 1) * sizeof (unsigned);

    codecvt_out.codeset_off = codecvt_out.wchar_off
        + charmap_.get_mb_cmap ().size () * 2 * sizeof (wchar_t);

    codecvt_out.charmap_off = codecvt_out.codeset_off
        + charmap_.get_code_set_name ().size () + 1 /* NUL */;
            
    const std::size_t mb_offset = codecvt_out.charmap_off
        + charmap_.get_charmap_name ().size () + 1 /* NUL */;

    // compute the size of narrow strings map which added to 
    // mb_offset will give the start of the transliteration data
    std::size_t xlit_data_offset = mb_offset;

    mb_cmap_iter iter;

    for (iter = charmap_.get_mb_cmap ().begin();
         iter != charmap_.get_mb_cmap().end(); ++iter) {
        xlit_data_offset += iter->first.size() + 1;
    }

    // now traverse again the utf8 tables for transliteration data
    // and recompute the offsets:
    const xlit_table_map_t::const_iterator xlit_table_map_end =
        xlit_table_map_.end ();

    xlit_table_map_t::iterator xit = xlit_table_map_.begin ();
    for (; xit != xlit_table_map_end; ++xit) {
        for (unsigned int i = 0; i < UCHAR_MAX + 1; ++i) {
            if (xit->second.offset_table [i] & 0x80000000)
                continue;
            // add the offset for xliteration data 
            xit->second.offset_table [i] += xlit_data_offset;
        }
    }

    codecvt_out.mb_cur_max = charmap_.get_mb_cur_max();

    issue_diag (I_OPENWR, false, 0, "writing %s\n", dir_name.c_str ());

    // create the stream with exceptions enabled
    std::ofstream out (dir_name.c_str(), std::ios::binary);
    out.exceptions (std::ios::failbit | std::ios::badbit);
            
    // write the codecvt_out structure
    out.write ((char*)&codecvt_out, sizeof codecvt_out);

    typedef codecvt_offsets_map_t::iterator off_iter_t;

    //////////////////////////////////////////////////////////////////
    // write out the multibyte to wchar_t tables
    issue_diag (I_WRITE, false, 0,
                "writing %lu multibyte tables (%lu characters)\n",
                mbchar_offs.size (), n_mbchars);

    for (off_iter_t it = mbchar_offs.begin (); it != mbchar_offs.end (); ++it) {
        for (unsigned i = 0; i <= UCHAR_MAX; ++i) {

            const unsigned off = it->second->off [i];

            out.write ((const char*)&off, sizeof off);
        }

        delete it->second;
    }

    // not needed beyond this point, clear it out
    mbchar_offs.clear ();

    //////////////////////////////////////////////////////////////////
    // write out the wchar_t to multibyte conversion tables
    issue_diag (I_WRITE, false, 0,
                "writing %lu wchar_t tables (%lu characters)\n",
                wchar_offs.size (), n_wchars);

    for (off_iter_t it = wchar_offs.begin (); it != wchar_offs.end (); ++it) {
        for (unsigned i = 0; i <= UCHAR_MAX; ++i) {

            // adjust offsets to multibyte characters (but not those
            // to other tables or invalid encodings)
            unsigned off = it->second->off [i];

            if (!(off & 0x80000000))
                off += mb_offset;

            out.write ((const char*)&off, sizeof off);
        }

        delete it->second;
    }
            
    // not needed beyond this point, clear it out
    wchar_offs.clear ();

    //////////////////////////////////////////////////////////////////
    // write out the UTF-8 to (libc) multibyte tables
    issue_diag (I_WRITE, false, 0,
                "writing %lu UTF-8 tables (%lu characters)\n",
                uchar_offs.size (), n_uchars);

    for (off_iter_t it = uchar_offs.begin (); it != uchar_offs.end (); ++it) {
        for (unsigned i = 0; i <= UCHAR_MAX; ++i) {

            // adjust offsets to multibyte characters (but not those
            // to other tables or invalid encodings)
            unsigned off = it->second->off [i];

            if (!(off & 0x80000000))
                off += mb_offset;

            out.write ((const char*)&off, sizeof off);
        }

        delete it->second;
    }

    // not needed beyond this point, clear it out
    uchar_offs.clear ();

    //////////////////////////////////////////////////////////////////
    // write out the transliteration UTF-8 lookup tables
    issue_diag (I_WRITE, false, 0,
                "writing transliteration table (size %lu)\n",
                xlit_table_map_.size ());

    xit = xlit_table_map_.begin ();
    for (; xit != xlit_table_map_end; ++xit) {
        const unsigned int* ptable = &xit->second.offset_table [0];
        for (unsigned int i = 0; i < UCHAR_MAX + 1; ++i, ++ptable) 
            out.write ((const char*)ptable, sizeof (unsigned int));
    }

    issue_diag (I_WRITE, false, 0,
                "writing the UCS table (%lu characters)\n",
                charmap_.get_mb_cmap ().size ());

    const mb_cmap_iter n_cmap2_end = charmap_.get_mb_cmap ().end ();

    // write the locale-encoded wchar_t and the UCS4 wchar_t
    for (iter = charmap_.get_mb_cmap ().begin();
         iter != n_cmap2_end; ++iter) {
        out.write ((const char*)&iter->second, sizeof (iter->second));
        out.write ((const char*)& (charmap_.get_ucs4_cmap().find 
                                   (charmap_.get_rw_cmap().find 
                                    (iter->second)->second))->second,
                   sizeof (wchar_t));
    }

    // write the code_set_name string and charmap string
    out << charmap_.get_code_set_name() << std::ends
        << charmap_.get_charmap_name() << std::ends;


    // write out the narrow character strings
    for (iter = charmap_.get_mb_cmap().begin();
         iter != n_cmap2_end; ++iter) {
        out.write (iter->first.c_str(), iter->first.size() + 1);
    }

    issue_diag (I_WRITE, false, 0,
                "writing transliteration data (size %lu)\n",
                xlit_map_.size ());

    // write out the transliteration data
    xlit_map_t::const_iterator xlit_data_it = xlit_map_.begin ();
    for (; xlit_data_it != xlit_map_.end (); ++xlit_data_it) {
        std::list<std::string>::const_iterator sit = 
            xlit_data_it->second.begin ();
        for (; sit != xlit_data_it->second.end (); ++sit) {
            out.write (sit->c_str (), sit->size () + 1);
        }
        out.write ("\0", 1);
    }
}