/**********************************************************************
// @@@ START COPYRIGHT @@@
//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//
// @@@ END COPYRIGHT @@@
**********************************************************************/
/* -*-C++-*-
 *****************************************************************************
 *
 * File:         conversionSJIS.cpp
 * RCS:          $Id: 
 * Description:  The implementation of SJIS related conversion routins
 *               
 *               
 * Created:      7/8/98
 * Modified:     $ $Date: 1998/08/10 16:00:50 $ (GMT)
 * Language:     C++
 * Status:       $State: Exp $
 *
 *
 *
 *
 *****************************************************************************
 */
// define MODULE_DEBUG when the module is to be debugged separately.
//#define MODULE_DEBUG

#include "NLSConversion.h"
#include "nawstring.h"


#ifdef MODULE_DEBUG
#include "stringBuf.cpp"
#endif


///////////////////////////////////////////////////////////////////////
// Standard SJIS code point ranges based on the book "Understanding 
// Japanese Information Processing" Table 4-9. pp. 73 and the massaged
// SJIS Unicode mapping table.  The original table is available from 
// unicode.org.
///////////////////////////////////////////////////////////////////////

inline Int32 in_range(Int32 x, Int32 lower, Int32 upper) 
{
   return ( lower <= x  &&  x <= upper ) ? 1 : 0;
}

// 1st block of single-byte characters in [0, 0x7F]
#define isSingleByteSJIS1stBlock(x) in_range(x, 0x00, 0x7F)

// 2nd block of single-byte characters in [0xA1, 0xDF]
#define isSingleByteSJIS2ndBlock(x) in_range(x, 0xA1, 0xDF)

typedef struct SJISCodeBound {
        unsigned char lower;
        unsigned char upper;
} SJISCodeBoundT;
      
// This function takes a SJIS string and returns its Unicode equivalent.
// The optional result argument holds the buffer into which the Unicode string
// will be stored. In case the result argument is NULL or the buffer it points
// at is not big enough, the function allocates memory from the heap
// (if the heap pointer is not NULL), or from the C run-time system heap
// and returns the result.
//
// If memory allocation fails, the function returns NULL.
// If any illegal characters are encountered, the function also returns NULL.
//
NAWcharBuf*
sjisToUnicode(const charBuf& sjisString, CollHeap *heap,
              NAWcharBuf*& result, NABoolean addNullAtEnd)
{

//
// These arrays are generated by the script /MiscVOB/i18n/sjis.ksh 
// and are copied into /sqlvob4/common. The seed file is 
// /MiscVOB/i18n/mx_sjis.txt which defines the Sjis to Unicode mapping. 
// Whenever a change is made to mx_sjis.txt, these arrays have to be
// regenerated by running sh sjis.ksh.
//
// Each of them (array_x for x in 0 to 4) represents the
// Unicode code values corresponding to blocks of SJIS code in a particular 
// range. The content of each array is included from a .h file whose name
// contains two hex numbers that defines the SJIS char range the array represents.
// Note the range is relevant to the leading byte of the SJIS characters only. 
// For example, sjis_81_84.h holds all Unicode chars mappable from SJIS in the 
// range from 0x81 to 0x84. 
//
// The trailing bytes of SJIS characters in each range run continuously 
// in strict ascending order, from 0x40 to 0xFC, except the "gap" characters
// at 0x7F. Such gap characters are excluded from these arrays because they are 
// not in SJIS.
//
// A few filler non-Unicode characters (0xFFFF) are purposely inserted in 
// these arrays to represent any un-defined characters in SJIS. These fillers 
// are useful in making fast algorithmic conversion possible through code point
// value manipulation. No filler character will be returned however.
//
// In each included file generated by the tool sjis.ksh, each line is in the
// format 
//     { /* SJIS code value */  Unicode code value }, /* remark */
// 
// For example
//     { /* 0x8140 */ 0x3000 }, /* IDEOGRAPHIC SPACE */
//
//
   static const NAWchar array_0[] = {
   #include "sjis_81_84.h"
   	};
   
   static const NAWchar array_1[] = {
   #include "sjis_87_9f.h"
   	};
   
   static const NAWchar array_2[] = {
   #include "sjis_e0_ea.h"
   	};
   
   static const NAWchar array_3[] = {
   #include "sjis_ed_ee.h"
   	};
   
   static const NAWchar array_4[] = {
   #include "sjis_fa_fc.h"
   	};
   
   // ranges determined by the leading byte
   static const SJISCodeBoundT SJISLeadByteBounds[] =
   {
      {0x81, 0x84}, { 0x87, 0x9F}, {0xE0, 0xEA}, {0xED, 0xEE}, {0xFA, 0xFC},
      {0x0, 0xFF } // catch all
   };

   // blocks determined by the trailing byte
   static const SJISCodeBoundT SJISTrailByteBounds[] =
   {
      {0x40, 0x7E}, {0x80, 0xFC}, 
      {0x0, 0xFF } // catch all
   };
 

   unsigned char* source = sjisString.data();
   Int32 sourceLen = sjisString.getStrLen();

   // the output Unicode string will have at most sjisString.length() 
   // characters. An extra char may be added depending on addNullAtEnd.
   NAWcharBuf* output = checkSpace(heap, sjisString.getStrLen(), result, addNullAtEnd);

   if ( output == 0 )
      return 0;

   NAWchar *base, *target;
   base = target = output -> data();

   unsigned char c, d;
   NAWchar u;
   Int32 i=0;

   while ( i < sourceLen ) {
		
      c = source[i++];

      if ( isSingleByteSJIS1stBlock(c) ) 
	u = (NAWchar)c;                 // found in the first single-byte block
      else 
      if ( isSingleByteSJIS2ndBlock(c) )
	u = (NAWchar)c - 0xA1 + 0xFF61; // found in the 2nd single-byte block
      else {

        // the second byte does not exist!
        if ( i == sjisString.getStrLen() ) {
          return 0;
        }

        // get the trailing byte
	d = source[i++];
         
        // is d a bad trailing byte?
        if ( d == 0x7F || d == 0xFD || d == 0xFE || d == 0xFF ) {
           return 0;
        }
        
        Int32 lead, trail;

        // find the range in which c is in.
        for ( lead = 0; lead<=4; lead++ ) 
           if ( in_range(c, SJISLeadByteBounds[lead].lower, 
                            SJISLeadByteBounds[lead].upper))
              break;

        // find the first or the second block for c.
        for (trail=0; trail<=2; trail++ ) 
           if ( in_range(d, SJISTrailByteBounds[trail].lower,
	                    SJISTrailByteBounds[trail].upper ))
              break;
    
        // out of range of SJIS legal code values
        if ( (c == 0xEA && d >= 0xA4) ||
             (c == 0xFC && d >= 0x4C) ||
             (lead == 5 ) || (trail == 2) 
           )
        {
	   return 0;
        }
         

        // Each chunk (all chars with identical lead byte)
        // has 16 * 18 = 192 chars. But because of the missing 
        // ones at 0x7F, 0xFD, 0xFE and 0xFF tailing byte, 
        // each chunk contains only 192 - 4 = 188 chars.
        //
        // In additional, a char with greater than 0x7F 
        // tailing byte (say, d) should be mapped to the entry 
        // with the index (d-1) instead of (d) in the right chunk. 
        // We do this by using the expression "- trail".
        switch (lead) {
            case 0:
	      u = array_0[(c-0x81) * 188 + d - 0x40 - trail];
              break;
            case 1:
	      u = array_1[(c-0x87) * 188 + d - 0x40 - trail];
              break;
            case 2:
	      u = array_2[(c-0xe0) * 188 + d - 0x40 - trail]; 
              break;
	    case 3:
	      u = array_3[(c-0xed) * 188 + d - 0x40 - trail];
              break;
            default:
              u = array_4[(c-0xfa) * 188 + d - 0x40 - trail];
	}
      }

      if ( u == 0xFFFF ) // filler chars are not defined in SJIS standard
         return 0;

      *target = u;
      target++;
   }
		
   Int32 finalLength = target-base;

   if ( addNullAtEnd == TRUE )
      (output -> data())[finalLength] = 0;

   output -> setStrLen(finalLength);
   return output;
}

typedef struct Unicode2SjisMap
{
   NAWchar Unicode;
   NAWchar sjis;
} Unicode2SjisMapT;

//
// Using the binary search method to find the SJIS code for a Unicode character
// contained in argument u. The SJIS code is returned in the argument sjis. The
// function returns TRUE if the conversion is sucessful, FALSE otherwise.
//
// The function only returns the SJIS that are double-byte.  Hense it is static.
//
static
NABoolean binarySearchU2STable(NAWchar u, NAWchar& sjis)
{


//
// This array contains the Unicode to SJIS mapping with. Each line in 
// the include file is in the format
//
// { Unicode_code_value, SJIS_code_value }, /* remark */
//
// It is assumed the Unicode_code_value column is in strict ascending order.
//
   static const Unicode2SjisMapT array_u2s[] = {
      #include "sjis_from_ucs2.h"
   };

   Int32 lowerLimit = 0;
   Int32 upperLimit = sizeof(array_u2s)/sizeof(Unicode2SjisMapT) - 1;
   Int32 middle = 0;

   do {
     middle = (lowerLimit + upperLimit ) / 2;

     if ( array_u2s[middle].Unicode == u ) {
        sjis = array_u2s[middle].sjis;
        return TRUE;
     }


     if ( array_u2s[middle].Unicode < u )
        lowerLimit = middle+1;
     else
        upperLimit = middle-1;

   } while ( lowerLimit <= upperLimit );

   return FALSE;
}

//
// Convert a single Unicode character (in argument wc) into SJIS multibyte
// format. The result is placed into the sjis argument. The number of SJIS 
// bytes is returned by the function. The function returns 0 if the 
// Unicode character is not mappable from SJIS.
//
Int32 unicodeToSjisChar(NAWchar wc, unsigned char *sjis, NABoolean allowInvalidCodePoint)
{
   NAWchar t;

   // single-byte SJIS characters, in [0, 0x7F]. The Unicode range is
   // also [0, 0x007F]
   if ( in_range(wc, 0, 0x007F) ) {
      sjis[0] = (unsigned char)wc;
      return 1; 
   }

   // single-byte SJIS characters, in [0xA1, 0xDF]. The Unicode range
   // is [0xFF61, 0xFF9F]
   if ( in_range(wc, 0xFF61, 0xFF9F) ) {
      sjis[0] = (unsigned char)(wc - 0xFF61 + 0xA1);
      return 1; 
   }

   // double byte
   if ( binarySearchU2STable(wc, t) && t >= 0x8140 ) {
      sjis[0] = (unsigned char)((t>>8) & (0xFF));
      sjis[1] = (unsigned char)(t & (0xFF));
      return 2;
   }

   // bad case
   if ( allowInvalidCodePoint == FALSE )
      return 0;
   else {
      sjis[0] = '?';
      return 1;
   }
}

// Unicode to SJIS conversion. 
//
// This function takes a Unicode string and returns its SJIS equivalent.
// The optional sjisString argument holds the buffer into which the Unicode string
// will be stored. In case the argument is NULL or it is not big enough, 
// the function allocates memory from the heap (if the heap pointer is not NULL), 
// or from the C run-time system heap.
// If the memory allocation fails, the function returns 0. If any illegal 
// characters are encountered, the function also returns 0.
//
charBuf* unicodeToSjis(const NAWcharBuf& unicodeString, CollHeap *heap,
                       charBuf*& sjisString, NABoolean addNullAtEnd,
                       NABoolean allowInvalidCodePoint)
{
   NAWchar* source = unicodeString.data();
   Int32 sourceLen = unicodeString.getStrLen();

   // the output Unicode string will have at most 2*unicodeString.length() 
   // characters. An extra char may be added depending on addNullAtEnd.
   charBuf* output = checkSpace(heap, BYTES_PER_NAWCHAR*unicodeString.getStrLen(), sjisString, addNullAtEnd);

   if ( output == 0 )
      return 0;
  
   unsigned char *base, *target;
   base = target = (unsigned char*) (output -> data());

   Int32 ct = 0;
   for ( Int32 i=0; i<sourceLen; i++ ) {

      ct = unicodeToSjisChar(source[i], target, allowInvalidCodePoint);

      if ( ct == 0 ) {
        if ( sjisString == NULL ) {
           if (heap)
             NADELETE(output, charBuf, heap);
           else
             delete output;
        }
        return NULL;
      } else
        target += ct;
   }

   Int32 finalLength = target - base;

   if ( addNullAtEnd == TRUE )
      (output -> data())[finalLength] = 0;

   output -> setStrLen(finalLength);
   return output;
}

#ifdef MODULE_DEBUG

//
// testing: dumping all Unicode codes for Sjis chars.
// Use the tool /MiscVOB/i18n/verify.pl to see the
// difference between the dumping and mx_sjis.txt.l2u.
// Should only see complaining about 0xFFFF.
//

static
Int32 leadbyte[] = {
        0x81, 0x82, 0x83, 0x84, 0x87, 0x88,
        0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e,
        0x8f,
        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 
        0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d,
        0x9e, 0x9f,
        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6,
        0xe7, 0xe8, 0xe9, 0xea, 0xed, 0xee, 0xfa, 
        0xfb, 0xfc
   };

void sjisDump()
{
   charBuf sjis(2);
   NAWcharBuf* unicode = new NAWcharBuf(10);

   for ( Int32 i=0; i<0xFF; i++ ) {
      sjis.data()[0] = i;
      sjis.setLength(1);

      unicode = sjisToUnicode(sjis, 0, unicode);
      if ( unicode )
         printf("0x%X  0x%X\n", i, unicode->data()[0]);
   }
	
   for ( i = 0; i<sizeof(leadbyte)/sizeof(Int32); i++ )
      for (Int32 j=0x40; j<=0xFF; j++ ) {
         sjis.data()[0] = leadbyte[i];
         sjis.data()[1] = j;
         sjis.setLength(2);

         unicode = sjisToUnicode(sjis, 0, unicode);
         if ( unicode )
            printf("0x%X%X  0x%X\n", leadbyte[i], j, unicode->data()[0]);
      }

   delete unicode;
}

// Roundtrip verification. 
//
// Perform a round-trip conversion testing for each valid SJIS.
//
// should not see any warnings.
//
void sjis2Unicode2sjis()
{
   printf("SJIS round-trip testing: ");

   charBuf sjis(2);
   NAWcharBuf* unicode = new NAWcharBuf(10);
   charBuf* remapped_sjis = new charBuf(10);

   // do the test for single-byte chars
   for ( Int32 i=0; i<0xDF; i++ ) {

      // skip non-SJIS chars
      if ( in_range(i, 0x80, 0xA0) )
         continue;

      sjis.data()[0] = i;
      sjis.setLength(1);

      unicode = sjisToUnicode(sjis, 0, unicode);

      if ( unicode == NULL ) {
         printf("SJIS to Unicode mapping failed: 0x%X.\n", i);
         return;
      }

      remapped_sjis = unicodeToSjis(*unicode, 0, remapped_sjis);

      if ( remapped_sjis == NULL ) {
        printf("Unicode 0x%X can not be mapped to SJIS\n", 
               unicode->data()[0]);
        return;
      }

      if ( remapped_sjis->data()[0] != sjis.data()[0] ) {
        printf("Roundtrip mapping failed: 0x%X  0x%X\n", 
               sjis.data()[0], remapped_sjis->data()[0]
              );
        return;
      }
   }
	
   // do the test for double-byte chars
   for ( i = 0; i<sizeof(leadbyte)/sizeof(Int32); i++ )
      for (Int32 j=0x40; j<=0xFF; j++ ) {

         sjis.data()[0] = leadbyte[i];
         sjis.data()[1] = j;
         sjis.setLength(2);

         unicode = sjisToUnicode(sjis, 0, unicode);
         if ( unicode == NULL ) {
            continue; // skip non-exist chars 
         }

         remapped_sjis = unicodeToSjis(*unicode, 0, remapped_sjis);

         if ( remapped_sjis == NULL ) {
            printf("Unicode 0x%X can not be mapped to SJIS\n", 
                   unicode->data()[0]);
            return;
         }

         if ( remapped_sjis->data()[0] != sjis.data()[0] || 
              remapped_sjis->data()[1] != sjis.data()[1] 
            ) {
           printf("Roundtrip mapping failed: sjis=0x%X%X  unicode=0x%X, remapped_sjis=0x%X%X\n", 
                  sjis.data()[0], sjis.data()[1],
                  unicode->data()[0],
                  remapped_sjis->data()[0], remapped_sjis->data()[1]
                 );
           return;
         }
      }

   delete unicode;
   delete remapped_sjis;

   printf("OK\n");
}

//
// Testing: assure all SJIS-mappable Unicode chars can be mapped
// to SJIS. Should not see any "Bad code ..." output.
//
void UCS2ToSjis()
{
   printf("Unicode to SJIS testing: ");

   NAWchar sjisChar;
   NAWcharBuf unicode(1);
   charBuf* sjis = new charBuf(10);

   Int32 n = sizeof(array_u2s)/sizeof(Unicode2SjisMapT);
   for ( Int32 i=0; i<n; i++ )
   {
      unicode.data()[0] = array_u2s[i].Unicode;
      unicodeToSjis(unicode, 0, sjis);

      switch (sjis->length()) {
         case 2:
            sjisChar = (NAWchar)(sjis->data()[1]);
            sjisChar |= (NAWchar)((sjis->data()[0] <<8));
            break;
         default:
            printf("Bad code 0x%X\n", array_u2s[i].Unicode);
            return;
      }
      if ( sjisChar != array_u2s[i].sjis ) {
         printf("Bad code 0x%x for 0x%X\n", sjisChar, array_u2s[i].Unicode);
         return;
      }
   }

   for ( i=0; i<=0x7F; i++ )
   {
     unicode.data()[0] = (NAWchar)i;
     unicodeToSjis(unicode, 0, sjis);

     switch (sjis->length()) {
        case 1:
           sjisChar = (NAWchar)sjis->data()[0];
           break;
        default:
           printf("Bad code 0x%X\n", i);
           return;
     }
     if ( sjisChar != i ) {
        printf("Bad code 0x%x for 0x%X\n", sjisChar, i);
        return;
     }
   }

   for ( i=0xFF61; i<=0xFF9F; i++ )
   {
     unicode.data()[0] = (NAWchar)i;
     unicodeToSjis(unicode, 0, sjis);

     switch (sjis->length()) {
        case 1:
           sjisChar = (NAWchar)sjis->data()[0];
           break;
        default:
           printf("Bad code 0x%X\n", i);
           return;
     }
     if ( sjisChar != i - 0xFF61 + 0xA1 ) {
        printf("Bad code 0x%x for 0x%X\n", sjisChar, i);
        return;
     }
   }
   printf("OK\n");
}

//
// Define MODULE_DEBUG at the beginning of this file and select one 
// of the following to test.  Build the test app with the command:
//
// cl /nologo /Zp4 /W3 /GX /Od /MDd /D "_DEBUG" /D "NA_WINNT" /Z7 \
// -o a.exe conversionsjis.cpp
//
Int32 main(Int32 argc, char** argv)
{
   //sjisDump();
   sjis2Unicode2sjis();
   UCS2ToSjis();
   return 0;
}

#endif