Utf.hpp

 
//
// TGUI - Texus' Graphical User Interface
// Copyright (C) 2012-2020 Bruno Van de Velde (vdv_b@tgui.eu)
//
// This software is provided 'as-is', without any express or implied warranty.
// In no event will the authors be held liable for any damages arising from the use of this software.
//
// Permission is granted to anyone to use this software for any purpose,
// including commercial applications, and to alter it and redistribute it freely,
// subject to the following restrictions:
//
// 1. The origin of this software must not be misrepresented;
// you must not claim that you wrote the original software.
// If you use this software in a product, an acknowledgment
// in the product documentation would be appreciated but is not required.
//
// 2. Altered source versions must be plainly marked as such,
// and must not be misrepresented as being the original software.
//
// 3. This notice may not be removed or altered from any source distribution.
//
 
 
#ifndef TGUI_UTF_HPP
#define TGUI_UTF_HPP
 
#include <string>
#include <array>
 
 
namespace tgui
{
 namespace utf
 {
 template <typename CharT> // CharT is either char or char8_t
 void encodeCharUtf8(char32_t input, std::basic_string<CharT>& outStrUtf8)
 {
 if (input < 128)
 {
 outStrUtf8.push_back(static_cast<CharT>(input));
 return;
 }
 
 // Encode the character (if it is valid)
 if ((input > 0x0010FFFF) || ((input >= 0xD800) && (input <= 0xDBFF)))
 return;
 
 // Get the number of bytes to write
 std::size_t bytestoWrite;
 if (input < 0x800)
 bytestoWrite = 2;
 else if (input < 0x10000)
 bytestoWrite = 3;
 else if (input <= 0x0010FFFF)
 bytestoWrite = 4;
 else
 return;
 
 static const std::uint8_t firstByteMask[5] = { 0, 0, 0xC0, 0xE0, 0xF0 };
 
 // Extract the bytes to write
 std::array<CharT, 4> bytes;
 if (bytestoWrite == 4) { bytes[3] = static_cast<CharT>((input | 0x80) & 0xBF); input >>= 6; }
 if (bytestoWrite >= 3) { bytes[2] = static_cast<CharT>((input | 0x80) & 0xBF); input >>= 6; }
 if (bytestoWrite >= 2) { bytes[1] = static_cast<CharT>((input | 0x80) & 0xBF); input >>= 6; }
 if (bytestoWrite >= 1) { bytes[0] = static_cast<CharT>(input | firstByteMask[bytestoWrite]); }
 
 // Add them to the output
 outStrUtf8.append(bytes.begin(), bytes.begin() + bytestoWrite);
 }
 
 
 template <typename CharIt> // CharIt is an iterator for a string containing either char or char8_t
 CharIt decodeCharUtf8(CharIt inputCharIt, CharIt inputEndIt, std::u32string& outStrUtf32)
 {
 if (static_cast<std::uint8_t>(*inputCharIt) < 128)
 {
 outStrUtf32.push_back(static_cast<char32_t>(static_cast<std::uint8_t>(*inputCharIt)));
 return ++inputCharIt;
 }
 
 // Some useful precomputed data
 static const std::uint32_t offsetsMap[6] = { 0x00000000, 0x00003080, 0x000E2080, 0x03C82080, 0xFA082080, 0x82082080 };
 static const std::uint8_t trailingMap[128] =
 {
 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5
 };
 
 // decode the character
 std::uint8_t trailingBytes = trailingMap[static_cast<std::uint8_t>(*inputCharIt) - 128];
 const std::uint32_t offset = offsetsMap[trailingBytes];
 if (inputCharIt + trailingBytes < inputEndIt)
 {
 char32_t outputChar = 0;
 while (trailingBytes > 0)
 {
 outputChar += static_cast<char32_t>(static_cast<std::uint8_t>(*inputCharIt++));
 outputChar <<= 6;
 --trailingBytes;
 }
 
 outputChar += static_cast<char32_t>(static_cast<std::uint8_t>(*inputCharIt++));
 outputChar -= offset;
 outStrUtf32.push_back(outputChar);
 }
 else // Incomplete character
 inputCharIt = inputEndIt;
 
 return inputCharIt;
 }
 
 
#if defined(__cpp_lib_char8_t) && (__cpp_lib_char8_t >= 201811L)
 inline std::u8string convertUtf32toUtf8(const std::u32string& strUtf32)
 {
 std::u8string outStrUtf8;
 outStrUtf8.reserve(strUtf32.length() + 1);
 for (const char32_t& codepoint : strUtf32)
 encodeCharUtf8(codepoint, outStrUtf8);
 
 return outStrUtf8;
 }
#endif
 
 template <typename CharIt>
 std::u32string convertUtf8toUtf32(CharIt inputBegin, CharIt inputEnd)
 {
 std::u32string outStrUtf32;
 outStrUtf32.reserve((inputEnd - inputBegin) + 1);
 
 auto it = inputBegin;
 while (it < inputEnd)
 it = decodeCharUtf8(it, inputEnd, outStrUtf32);
 
 return outStrUtf32;
 }
 
 
 template <typename U16CharIt>
 std::u32string convertUtf16toUtf32(U16CharIt inputBegin, U16CharIt inputEnd)
 {
 std::u32string outStrUtf32;
 outStrUtf32.reserve((inputEnd - inputBegin) + 1);
 
 auto it = inputBegin;
 while (it < inputEnd)
 {
 const char16_t first = *it++;
 
 // Copy the character if it isn't a surrogate pair
 if ((first < 0xD800) || (first > 0xDBFF))
 {
 outStrUtf32.push_back(static_cast<char32_t>(first));
 continue;
 }
 
 // We need to read another character
 if (it == inputEnd)
 break;
 
 const char16_t second = *it++;
 if ((second >= 0xDC00) && (second <= 0xDFFF))
 outStrUtf32.push_back(((static_cast<char32_t>(first) - 0xD800) << 10) + (static_cast<char32_t>(second) - 0xDC00) + 0x0010000);
 }
 
 
 return outStrUtf32;
 }
 
 
 template <typename WCharIt>
 std::u32string convertWidetoUtf32(WCharIt inputBegin, WCharIt inputEnd)
 {
 std::u32string outStrUtf32;
 outStrUtf32.reserve((inputEnd - inputBegin) + 1);
 
 // std::wstring uses UCS-2 on Windows and UCS-4 on unix, so we can be cast directly
 for (auto it = inputBegin; it != inputEnd; ++it)
 outStrUtf32.push_back(static_cast<char32_t>(*it));
 
 
 return outStrUtf32;
 }
 
 
 inline std::string convertUtf32toLatin1(const std::u32string& strUtf32)
 {
 std::string outStr;
 outStr.reserve(strUtf32.length() + 1);
 for (const char32_t codepoint : strUtf32)
 {
 if (codepoint < 256)
 outStr.push_back(static_cast<char>(codepoint));
 }
 
 return outStr;
 }
 
 
 inline std::string convertUtf32toStdStringUtf8(const std::u32string& strUtf32)
 {
 std::string outStrUtf8;
 outStrUtf8.reserve(strUtf32.length() + 1);
 for (const char32_t codepoint : strUtf32)
 encodeCharUtf8(codepoint, outStrUtf8);
 
 return outStrUtf8;
 }
 
 
 inline std::wstring convertUtf32toWide(const std::u32string& strUtf32)
 {
 std::wstring outStr;
 outStr.reserve(strUtf32.length() + 1);
 
 
#if defined(__cpp_if_constexpr) && (__cpp_if_constexpr >= 201606L)
 if constexpr (sizeof(wchar_t) == 4)
#else
 #if defined TGUI_SYSTEM_WINDOWS && defined _MSC_VER
 #pragma warning(push)
 #pragma warning(disable:4127)
 #endif
 if (sizeof(wchar_t) == 4)
 #if defined TGUI_SYSTEM_WINDOWS && defined _MSC_VER
 #pragma warning(pop)
 #endif
#endif
 {
 // On Unix, wide characters are UCS-4 and we can just copy the characters
 for (const char32_t codepoint : strUtf32)
 outStr.push_back(static_cast<wchar_t>(codepoint));
 }
 else
 {
 // On Windows, wide characters are UCS-2. We just drop the characters that don't fit within a single wide character here.
 for (const char32_t codepoint : strUtf32)
 {
 if ((codepoint < 0xD800) || ((codepoint > 0xDFFF) && (codepoint <= 0xFFFF)))
 outStr.push_back(static_cast<wchar_t>(codepoint));
 }
 }
 
 return outStr;
 }
 
 
 inline std::u16string convertUtf32toUtf16(const std::u32string& strUtf32)
 {
 std::u16string outStrUtf16;
 outStrUtf16.reserve(strUtf32.length() + 1);
 
 for (const char32_t codepoint : strUtf32)
 {
 // If the codepoint fitst inside 2 bytes and it would represent a valid character then just copy it
 if (codepoint <= 0xFFFF)
 {
 if ((codepoint < 0xD800) || (codepoint > 0xDFFF))
 outStrUtf16.push_back(static_cast<char16_t>(codepoint));
 
 continue;
 }
 else if (codepoint > 0x0010FFFF)
 continue; // Invalid character (greater than the maximum Unicode value)
 
 // The input character needs be converted to two UTF-16 elements
 outStrUtf16.push_back(static_cast<char16_t>(((codepoint - 0x0010000) >> 10) + 0xD800));
 outStrUtf16.push_back(static_cast<char16_t>(((codepoint - 0x0010000) & 0x3FFUL) + 0xDC00));
 }
 
 return outStrUtf16;
 }
 
 }
}
 
 
#endif // TGUI_UTF_HPP