src/main/cpp/charsetdecoder.cpp

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#define NOMINMAX /* tell windows not to define min/max macros */
#include <log4cxx/private/string_c11.h>
#include <log4cxx/logstring.h>
#include <log4cxx/helpers/charsetdecoder.h>
#include <log4cxx/private/bytebuffer_priv.h>
#include <log4cxx/helpers/exception.h>
#include <log4cxx/helpers/pool.h>
#include <log4cxx/helpers/loglog.h>
#include <apr_xlate.h>
#if !defined(LOG4CXX)
	#define LOG4CXX 1
#endif
#include <log4cxx/private/log4cxx_private.h>
#include <locale.h>
#include <apr_portable.h>
#include <log4cxx/helpers/stringhelper.h>
#include <log4cxx/helpers/transcoder.h>
#include <mutex>

using namespace LOG4CXX_NS;
using namespace LOG4CXX_NS::helpers;

IMPLEMENT_LOG4CXX_OBJECT(CharsetDecoder)


namespace LOG4CXX_NS
{
namespace helpers
{

#if APR_HAS_XLATE
/**
 *  Converts from an arbitrary encoding to LogString
 *    using apr_xlate.  Requires real iconv implementation,
*    apr-iconv will crash in use.
 */
class APRCharsetDecoder : public CharsetDecoder
{
	public:
		/**
		 *  Creates a new instance.
		 *  @param frompage name of source encoding.
		 */
		APRCharsetDecoder(const LogString& frompage) : pool()
		{
#if LOG4CXX_LOGCHAR_IS_WCHAR
			const char* topage = "WCHAR_T";
#endif
#if LOG4CXX_LOGCHAR_IS_UTF8
			const char* topage = "UTF-8";
#endif
			std::string fpage(Transcoder::encodeCharsetName(frompage));
			apr_status_t stat = apr_xlate_open(&convset,
					topage,
					fpage.c_str(),
					pool.getAPRPool());

			if (stat != APR_SUCCESS)
			{
				throw IllegalArgumentException(frompage);
			}
		}

		/**
		 *  Destructor.
		 */
		virtual ~APRCharsetDecoder()
		{
		}

		virtual log4cxx_status_t decode(ByteBuffer& in_param,
			LogString& out)
		{
			auto& in = in_param.impl();
			enum { BUFSIZE = 256 };
			logchar buf[BUFSIZE];
			const apr_size_t initial_outbytes_left = BUFSIZE * sizeof(logchar);
			apr_status_t stat = APR_SUCCESS;

			if (in.remaining() == 0)
			{
				size_t outbytes_left = initial_outbytes_left;
				{
					std::lock_guard<std::mutex> lock(mutex);
					stat = apr_xlate_conv_buffer((apr_xlate_t*) convset,
							NULL, NULL, (char*) buf, &outbytes_left);
				}
				out.append(buf, (initial_outbytes_left - outbytes_left) / sizeof(logchar));
			}
			else
			{
				while (in.remaining() > 0 && stat == APR_SUCCESS)
				{
					size_t inbytes_left = in.remaining();
					size_t initial_inbytes_left = inbytes_left;
					apr_size_t outbytes_left = initial_outbytes_left;
					{
						std::lock_guard<std::mutex> lock(mutex);
						stat = apr_xlate_conv_buffer((apr_xlate_t*) convset,
								in.current(),
								&inbytes_left,
								(char*) buf,
								&outbytes_left);
					}
					out.append(buf, (initial_outbytes_left - outbytes_left) / sizeof(logchar));
					if (inbytes_left == initial_inbytes_left && stat == APR_SUCCESS)
					{
						stat = APR_BADCH;
						break;
					}
					in.increment_position(initial_inbytes_left - inbytes_left);
				}
			}

			return stat;
		}

	private:
		APRCharsetDecoder(const APRCharsetDecoder&);
		APRCharsetDecoder& operator=(const APRCharsetDecoder&);
		LOG4CXX_NS::helpers::Pool pool;
		std::mutex mutex;
		apr_xlate_t* convset;
};

#endif

#if LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_MBSRTOWCS
/**
*    Converts from the default multi-byte string to
*        LogString using mbstowcs.
*
*/
class MbstowcsCharsetDecoder : public CharsetDecoder
{
	public:
		MbstowcsCharsetDecoder()
		{
		}

		virtual ~MbstowcsCharsetDecoder()
		{
		}

	private:
		inline log4cxx_status_t append(LogString& out, const wchar_t* buf)
		{
			out.append(buf);
			return APR_SUCCESS;
		}

		virtual log4cxx_status_t decode(ByteBuffer& in_param,
			LogString& out)
		{
			auto& in = in_param.impl();
			log4cxx_status_t stat = APR_SUCCESS;
			enum { BUFSIZE = 256 };
			wchar_t wbuf[BUFSIZE];
			char cbuf[BUFSIZE*4];

			mbstate_t mbstate;
			memset(&mbstate, 0, sizeof(mbstate));

			while (in.remaining() > 0)
			{
				const char* src = in.current();

				if (*src == 0)
				{
					out.append(1, (logchar) 0);
					in.increment_position(1);
				}
				else
				{
					auto available = std::min(sizeof (cbuf) - 1, in.remaining());
					strncpy(cbuf, src, available);
					cbuf[available] = 0;
					src = cbuf;
					size_t wCharCount = mbsrtowcs(wbuf,
							&src,
							BUFSIZE - 1,
							&mbstate);
					// mbsrtowcs sets *src to nullptr when it consumes a null wide character.
					// Performing pointer arithmetic on that nullptr (src - cbuf) is undefined
					// behaviour, so recover the consumed byte count from the position of the
					// null that stopped the conversion instead.
					size_t converted;
					if (src == nullptr)
					{
						size_t nullPos = 0;
						while (nullPos < available && cbuf[nullPos] != 0)
						{
							++nullPos;
						}
						// If the null came from the input bytes, it was consumed too;
						// if it is the sentinel we wrote at cbuf[available], stop at available.
						converted = (nullPos < available) ? nullPos + 1 : available;
					}
					else
					{
						converted = static_cast<size_t>(src - cbuf);
					}
					in.increment_position(converted);

					if (wCharCount == (size_t) -1) // Illegal byte sequence?
					{
						LogString msg(LOG4CXX_STR("Illegal byte sequence at "));
						msg.append(std::to_wstring(in.position()));
						msg.append(LOG4CXX_STR(" of "));
						msg.append(std::to_wstring(in.limit()));
						LogLog::warn(msg);
						stat = APR_BADCH;
						break;
					}
					else
					{
						// FIX: Check for incomplete sequence infinite loop.
						// If mbsrtowcs returns success (>=0) but converted 0 bytes while data remains,
						// we are stuck (e.g. incomplete multibyte char at EOF).
						if (converted == 0 && in.remaining() > 0)
						{
							LogString msg(LOG4CXX_STR("Incomplete multibyte sequence at end of buffer"));
							LogLog::warn(msg);
							stat = APR_BADCH;
							break; // Break the infinite loop
						}

						wbuf[wCharCount] = 0;
						stat = append(out, wbuf);
					}
				}
			}

			return stat;
		}



	private:
		MbstowcsCharsetDecoder(const MbstowcsCharsetDecoder&);
		MbstowcsCharsetDecoder& operator=(const MbstowcsCharsetDecoder&);
};
#endif


/**
*    Decoder used when the external and internal charsets
*    are the same.
*
*/
class TrivialCharsetDecoder : public CharsetDecoder
{
	public:
		TrivialCharsetDecoder()
		{
		}

		virtual ~TrivialCharsetDecoder()
		{
		}

		virtual log4cxx_status_t decode(ByteBuffer& in_param,
			LogString& out)
		{
			auto& in = in_param.impl();
			size_t remaining = in.remaining();

			if ( remaining > 0)
			{
				auto src = in.current();
				auto count = remaining / sizeof(logchar);
				out.append(reinterpret_cast<const logchar*>(src), count);
				in.increment_position(remaining);
			}

			return APR_SUCCESS;
		}



	private:
		TrivialCharsetDecoder(const TrivialCharsetDecoder&);
		TrivialCharsetDecoder& operator=(const TrivialCharsetDecoder&);
};

/**
*    Converts from UTF-8 to LogString
*
*/
class UTF8CharsetDecoder : public CharsetDecoder
{
	public:
		UTF8CharsetDecoder()
		{
		}

		virtual ~UTF8CharsetDecoder()
		{
		}

	private:
		virtual log4cxx_status_t decode(ByteBuffer& in_param,
			LogString& out)
		{
			auto& in = in_param.impl();
			auto availableByteCount = in.remaining();
			while (0 < availableByteCount)
			{
				auto sv = getUTF8CodePoint(in);
				auto nextAvailableByteCount = in.remaining();
				if (sv == 0xFFFF || nextAvailableByteCount == availableByteCount)
					return APR_BADCH;
				Transcoder::encode(sv, out);
				availableByteCount = nextAvailableByteCount;
			}
			return APR_SUCCESS;
		}

	private:
		UTF8CharsetDecoder(const UTF8CharsetDecoder&);
		UTF8CharsetDecoder& operator=(const UTF8CharsetDecoder&);
};

/**
*    Converts from ISO-8859-1 to LogString.
*
*/
class ISOLatinCharsetDecoder : public CharsetDecoder
{
	public:
		ISOLatinCharsetDecoder()
		{
		}

		virtual ~ISOLatinCharsetDecoder()
		{
		}

	private:
		virtual log4cxx_status_t decode(ByteBuffer& in_param,
			LogString& out)
		{
			auto& in = in_param.impl();
			auto availableByteCount = in.remaining();
			auto src = in.current();
			auto srcEnd = src + availableByteCount;

			while (src < srcEnd)
			{
				auto sv = static_cast<unsigned int>(static_cast<unsigned char>(*src++));
				Transcoder::encode(sv, out);
			}
			in.increment_position(availableByteCount);

			return APR_SUCCESS;
		}



	private:
		ISOLatinCharsetDecoder(const ISOLatinCharsetDecoder&);
		ISOLatinCharsetDecoder& operator=(const ISOLatinCharsetDecoder&);
};


/**
*    Converts from US-ASCII to LogString.
*
*/
class USASCIICharsetDecoder : public CharsetDecoder
{
	public:
		USASCIICharsetDecoder()
		{
		}

		virtual ~USASCIICharsetDecoder()
		{
		}

	private:

		virtual log4cxx_status_t decode(ByteBuffer& in_param,
			LogString& out)
		{
			auto& in = in_param.impl();
			log4cxx_status_t stat = APR_SUCCESS;

			auto availableByteCount = in.remaining();
			auto src = in.current();
			auto srcEnd = src + availableByteCount;
			size_t byteCount = 0;
			while (src < srcEnd)
			{
				auto sv = static_cast<unsigned int>(*src++);

				if (sv < 0x80)
				{
					++byteCount;
					Transcoder::encode(sv, out);
				}
				else
				{
					stat = APR_BADCH;
					break;
				}
			}
			in.increment_position(byteCount);

			return stat;
		}



	private:
		USASCIICharsetDecoder(const USASCIICharsetDecoder&);
		USASCIICharsetDecoder& operator=(const USASCIICharsetDecoder&);
};

/**
 *    Charset decoder that uses current locale settings.
 */
class LocaleCharsetDecoder : public CharsetDecoder
{
	public:
		LocaleCharsetDecoder() : state()
		{
		}
		log4cxx_status_t decode(ByteBuffer& in_param, LogString& out) override
		{
			auto& in = in_param.impl();
			log4cxx_status_t result = APR_SUCCESS;
			auto p = in.current();
			auto availableByteCount = in.remaining();
			size_t byteCount = 0;
#if !LOG4CXX_CHARSET_EBCDIC
			if (std::mbsinit(&this->state)) // ByteBuffer not partially decoded?
			{
				// Copy single byte characters
				for (; byteCount < availableByteCount && static_cast<unsigned int>(*p) < 0x80; ++byteCount, ++p)
				{
					out.append(1, *p);
				}
			}
#endif
			// Decode characters that may be represented by multiple bytes
			while (byteCount < availableByteCount)
			{
				wchar_t ch = 0;
				size_t n = std::mbrtowc(&ch, p, availableByteCount - byteCount, &this->state);
				if (0 == n) // NULL encountered?
				{
					++byteCount;
					break;
				}
				if (static_cast<std::size_t>(-1) == n) // decoding error?
				{
					result = APR_BADCH;
					break;
				}
				if (static_cast<std::size_t>(-2) == n) // incomplete sequence?
				{
					break;
				}
				Transcoder::encode(static_cast<unsigned int>(ch), out);
				byteCount += n;
				p += n;
			}
			in.increment_position(byteCount);
			return result;
		}

	private:
		std::mbstate_t state;
};



} // namespace helpers

}  //namespace log4cxx


CharsetDecoder::CharsetDecoder()
{
}


CharsetDecoder::~CharsetDecoder()
{
}

CharsetDecoder* CharsetDecoder::createDefaultDecoder()
{
#if LOG4CXX_CHARSET_UTF8
#if LOG4CXX_LOGCHAR_IS_UTF8
	return new TrivialCharsetDecoder();
#else
	return new UTF8CharsetDecoder();
#endif
#elif LOG4CXX_CHARSET_ISO88591 || defined(_WIN32_WCE)
	return new ISOLatinCharsetDecoder();
#elif LOG4CXX_CHARSET_USASCII
	return new USASCIICharsetDecoder();
#elif LOG4CXX_LOGCHAR_IS_WCHAR && LOG4CXX_HAS_MBSRTOWCS
	return new MbstowcsCharsetDecoder();
#else
	return new LocaleCharsetDecoder();
#endif
}

CharsetDecoderPtr CharsetDecoder::getDefaultDecoder()
{
	static WideLife<CharsetDecoderPtr> decoder(createDefaultDecoder());

	//
	//  if invoked after static variable destruction
	//     (if logging is called in the destructor of a static object)
	//     then create a new decoder.
	//
	if (decoder.value() == 0)
	{
		return CharsetDecoderPtr( createDefaultDecoder() );
	}

	return decoder;
}

CharsetDecoderPtr CharsetDecoder::getUTF8Decoder()
{
	return std::make_shared<UTF8CharsetDecoder>();
}

CharsetDecoderPtr CharsetDecoder::getISOLatinDecoder()
{
	return std::make_shared<ISOLatinCharsetDecoder>();
}


CharsetDecoderPtr CharsetDecoder::getDecoder(const LogString& charset)
{
	if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("UTF-8"), LOG4CXX_STR("utf-8")) ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("UTF8"), LOG4CXX_STR("utf8")) ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("CP65001"), LOG4CXX_STR("cp65001")))
	{
#if LOG4CXX_LOGCHAR_IS_UTF8
		return std::make_shared<TrivialCharsetDecoder>();
#else
		return std::make_shared<UTF8CharsetDecoder>();
#endif
	}
	else if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("C"), LOG4CXX_STR("c")) ||
		charset == LOG4CXX_STR("646") ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("US-ASCII"), LOG4CXX_STR("us-ascii")) ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("ISO646-US"), LOG4CXX_STR("iso646-US")) ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("ANSI_X3.4-1968"), LOG4CXX_STR("ansi_x3.4-1968")) ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("CP20127"), LOG4CXX_STR("cp20127")))
	{
		return std::make_shared<USASCIICharsetDecoder>();
	}
	else if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("ISO-8859-1"), LOG4CXX_STR("iso-8859-1")) ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("ISO-LATIN-1"), LOG4CXX_STR("iso-latin-1")) ||
		StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("CP1252"), LOG4CXX_STR("cp1252")))
	{
		return std::make_shared<ISOLatinCharsetDecoder>();
	}
	else if (StringHelper::equalsIgnoreCase(charset, LOG4CXX_STR("LOCALE"), LOG4CXX_STR("locale")))
	{
		return std::make_shared<LocaleCharsetDecoder>();
	}

#if APR_HAS_XLATE
	return std::make_shared<APRCharsetDecoder>(charset);
#else
	throw IllegalArgumentException(charset);
#endif
}

log4cxx_status_t CharsetDecoder::decode(const char* in, size_t maxByteCount, LogString& out)
{
	ByteBuffer buf((char*)in, strnlen_s(in, maxByteCount));
	return decode(buf, out);
}

unsigned int CharsetDecoder::getUTF8CodePoint(ByteBufferPriv& in)
{
	auto availableByteCount = in.remaining();
	if (0 == availableByteCount)
		return 0xFFFF;

	auto pChar = in.current();
	auto ch1 = static_cast<unsigned char>(*pChar);
	if (ch1 <= 0x7F)
	{
		in.increment_position(1);
		return ch1;
	}

	//
	//   should not have continuation character here
	//
	if ((ch1 & 0xC0) != 0x80 && 1 < availableByteCount)
	{
		auto ch2 = static_cast<unsigned char>(*(pChar + 1));
		if ((ch2 & 0xC0) != 0x80) // not a continuation?
			return 0xFFFF;

		if ((ch1 & 0xE0) == 0xC0)
		{
			unsigned int rv = ((ch1 & 0x1F) << 6) + (ch2 & 0x3F);
			if (rv >= 0x80)
			{
				in.increment_position(2);
				return rv;
			}
			return 0xFFFF;
		}

		if (2 < availableByteCount)
		{
			auto ch3 = static_cast<unsigned char>(*(pChar + 2));
			if ((ch3 & 0xC0) != 0x80) // not a continuation?
				return 0xFFFF;

			if ((ch1 & 0xF0) == 0xE0)
			{
				unsigned int rv = ((ch1 & 0x0F) << 12)
					+ ((ch2 & 0x3F) << 6)
					+ (ch3 & 0x3F);

				// RFC 3629 §3 prohibits UTF-8 encodings of the UTF-16 surrogate
				// halves (U+D800..U+DFFF); accepting them lets malformed Unicode
				// cross the decode boundary into LogString and downstream output.
				if (rv < 0x800 || (0xD800 <= rv && rv <= 0xDFFF))
					return 0xFFFF;

				in.increment_position(3);
				return rv;
			}

			if (3 < availableByteCount)
			{
				auto ch4 = static_cast<unsigned char>(*(pChar + 3));
				if ((ch4 & 0xC0) != 0x80) // not a continuation?
					return 0xFFFF;

				unsigned int rv = ((ch1 & 0x07) << 18)
					+ ((ch2 & 0x3F) << 12)
					+ ((ch3 & 0x3F) << 6)
					+ (ch4 & 0x3F);

				// RFC 3629 §3 caps UTF-8 at U+10FFFF; lead bytes F5..F7 (and
				// F4 with an over-high trailer) produce rv > 0x10FFFF, which
				// is not a Unicode code point. Without this bound, encodeUTF16
				// later silently aliases the bogus value to a valid in-range
				// code point — a substitution-collision filter-bypass primitive.
				// Lead bytes F8..FF are never valid UTF-8, but the & 0x07 mask
				// discards their high bits, so without the (ch1 & 0xF8) == 0xF0
				// guard F8 BF BF BF would alias to U+3FFFF instead of being
				// rejected.
				if ((ch1 & 0xF8) == 0xF0 && rv > 0xFFFF && rv <= 0x10FFFF)
				{
					in.increment_position(4);
					return rv;
				}

			}
		}
	}
	return 0xFFFF;
}