/home/arjun/llvm-project/llvm/lib/Support/ConvertUTFWrapper.cpp
| Line | Count | Source (jump to first uncovered line) | 
| 1 |  | //===-- ConvertUTFWrapper.cpp - Wrap ConvertUTF.h with clang data types -----=== | 
| 2 |  | // | 
| 3 |  | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | 
| 4 |  | // See https://llvm.org/LICENSE.txt for license information. | 
| 5 |  | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | 
| 6 |  | // | 
| 7 |  | //===----------------------------------------------------------------------===// | 
| 8 |  |  | 
| 9 |  | #include "llvm/ADT/ArrayRef.h" | 
| 10 |  | #include "llvm/ADT/StringRef.h" | 
| 11 |  | #include "llvm/Support/ConvertUTF.h" | 
| 12 |  | #include "llvm/Support/ErrorHandling.h" | 
| 13 |  | #include "llvm/Support/SwapByteOrder.h" | 
| 14 |  | #include <string> | 
| 15 |  | #include <vector> | 
| 16 |  |  | 
| 17 |  | namespace llvm { | 
| 18 |  |  | 
| 19 |  | bool ConvertUTF8toWide(unsigned WideCharWidth, llvm::StringRef Source, | 
| 20 | 0 |                        char *&ResultPtr, const UTF8 *&ErrorPtr) { | 
| 21 | 0 |   assert(WideCharWidth == 1 || WideCharWidth == 2 || WideCharWidth == 4); | 
| 22 | 0 |   ConversionResult result = conversionOK; | 
| 23 | 0 |   // Copy the character span over. | 
| 24 | 0 |   if (WideCharWidth == 1) { | 
| 25 | 0 |     const UTF8 *Pos = reinterpret_cast<const UTF8*>(Source.begin()); | 
| 26 | 0 |     if (!isLegalUTF8String(&Pos, reinterpret_cast<const UTF8*>(Source.end()))) { | 
| 27 | 0 |       result = sourceIllegal; | 
| 28 | 0 |       ErrorPtr = Pos; | 
| 29 | 0 |     } else { | 
| 30 | 0 |       memcpy(ResultPtr, Source.data(), Source.size()); | 
| 31 | 0 |       ResultPtr += Source.size(); | 
| 32 | 0 |     } | 
| 33 | 0 |   } else if (WideCharWidth == 2) { | 
| 34 | 0 |     const UTF8 *sourceStart = (const UTF8*)Source.data(); | 
| 35 | 0 |     // FIXME: Make the type of the result buffer correct instead of | 
| 36 | 0 |     // using reinterpret_cast. | 
| 37 | 0 |     UTF16 *targetStart = reinterpret_cast<UTF16*>(ResultPtr); | 
| 38 | 0 |     ConversionFlags flags = strictConversion; | 
| 39 | 0 |     result = ConvertUTF8toUTF16( | 
| 40 | 0 |         &sourceStart, sourceStart + Source.size(), | 
| 41 | 0 |         &targetStart, targetStart + Source.size(), flags); | 
| 42 | 0 |     if (result == conversionOK) | 
| 43 | 0 |       ResultPtr = reinterpret_cast<char*>(targetStart); | 
| 44 | 0 |     else | 
| 45 | 0 |       ErrorPtr = sourceStart; | 
| 46 | 0 |   } else if (WideCharWidth == 4) { | 
| 47 | 0 |     const UTF8 *sourceStart = (const UTF8*)Source.data(); | 
| 48 | 0 |     // FIXME: Make the type of the result buffer correct instead of | 
| 49 | 0 |     // using reinterpret_cast. | 
| 50 | 0 |     UTF32 *targetStart = reinterpret_cast<UTF32*>(ResultPtr); | 
| 51 | 0 |     ConversionFlags flags = strictConversion; | 
| 52 | 0 |     result = ConvertUTF8toUTF32( | 
| 53 | 0 |         &sourceStart, sourceStart + Source.size(), | 
| 54 | 0 |         &targetStart, targetStart + Source.size(), flags); | 
| 55 | 0 |     if (result == conversionOK) | 
| 56 | 0 |       ResultPtr = reinterpret_cast<char*>(targetStart); | 
| 57 | 0 |     else | 
| 58 | 0 |       ErrorPtr = sourceStart; | 
| 59 | 0 |   } | 
| 60 | 0 |   assert((result != targetExhausted) | 
| 61 | 0 |          && "ConvertUTF8toUTFXX exhausted target buffer"); | 
| 62 | 0 |   return result == conversionOK; | 
| 63 | 0 | } | 
| 64 |  |  | 
| 65 | 0 | bool ConvertCodePointToUTF8(unsigned Source, char *&ResultPtr) { | 
| 66 | 0 |   const UTF32 *SourceStart = &Source; | 
| 67 | 0 |   const UTF32 *SourceEnd = SourceStart + 1; | 
| 68 | 0 |   UTF8 *TargetStart = reinterpret_cast<UTF8 *>(ResultPtr); | 
| 69 | 0 |   UTF8 *TargetEnd = TargetStart + 4; | 
| 70 | 0 |   ConversionResult CR = ConvertUTF32toUTF8(&SourceStart, SourceEnd, | 
| 71 | 0 |                                            &TargetStart, TargetEnd, | 
| 72 | 0 |                                            strictConversion); | 
| 73 | 0 |   if (CR != conversionOK) | 
| 74 | 0 |     return false; | 
| 75 | 0 |  | 
| 76 | 0 |   ResultPtr = reinterpret_cast<char*>(TargetStart); | 
| 77 | 0 |   return true; | 
| 78 | 0 | } | 
| 79 |  |  | 
| 80 | 0 | bool hasUTF16ByteOrderMark(ArrayRef<char> S) { | 
| 81 | 0 |   return (S.size() >= 2 && | 
| 82 | 0 |           ((S[0] == '\xff' && S[1] == '\xfe') || | 
| 83 | 0 |            (S[0] == '\xfe' && S[1] == '\xff'))); | 
| 84 | 0 | } | 
| 85 |  |  | 
| 86 | 0 | bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) { | 
| 87 | 0 |   assert(Out.empty()); | 
| 88 | 0 | 
 | 
| 89 | 0 |   // Error out on an uneven byte count. | 
| 90 | 0 |   if (SrcBytes.size() % 2) | 
| 91 | 0 |     return false; | 
| 92 | 0 |  | 
| 93 | 0 |   // Avoid OOB by returning early on empty input. | 
| 94 | 0 |   if (SrcBytes.empty()) | 
| 95 | 0 |     return true; | 
| 96 | 0 |  | 
| 97 | 0 |   const UTF16 *Src = reinterpret_cast<const UTF16 *>(SrcBytes.begin()); | 
| 98 | 0 |   const UTF16 *SrcEnd = reinterpret_cast<const UTF16 *>(SrcBytes.end()); | 
| 99 | 0 | 
 | 
| 100 | 0 |   // Byteswap if necessary. | 
| 101 | 0 |   std::vector<UTF16> ByteSwapped; | 
| 102 | 0 |   if (Src[0] == UNI_UTF16_BYTE_ORDER_MARK_SWAPPED) { | 
| 103 | 0 |     ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd); | 
| 104 | 0 |     for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I) | 
| 105 | 0 |       ByteSwapped[I] = llvm::ByteSwap_16(ByteSwapped[I]); | 
| 106 | 0 |     Src = &ByteSwapped[0]; | 
| 107 | 0 |     SrcEnd = &ByteSwapped[ByteSwapped.size() - 1] + 1; | 
| 108 | 0 |   } | 
| 109 | 0 | 
 | 
| 110 | 0 |   // Skip the BOM for conversion. | 
| 111 | 0 |   if (Src[0] == UNI_UTF16_BYTE_ORDER_MARK_NATIVE) | 
| 112 | 0 |     Src++; | 
| 113 | 0 | 
 | 
| 114 | 0 |   // Just allocate enough space up front.  We'll shrink it later.  Allocate | 
| 115 | 0 |   // enough that we can fit a null terminator without reallocating. | 
| 116 | 0 |   Out.resize(SrcBytes.size() * UNI_MAX_UTF8_BYTES_PER_CODE_POINT + 1); | 
| 117 | 0 |   UTF8 *Dst = reinterpret_cast<UTF8 *>(&Out[0]); | 
| 118 | 0 |   UTF8 *DstEnd = Dst + Out.size(); | 
| 119 | 0 | 
 | 
| 120 | 0 |   ConversionResult CR = | 
| 121 | 0 |       ConvertUTF16toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion); | 
| 122 | 0 |   assert(CR != targetExhausted); | 
| 123 | 0 | 
 | 
| 124 | 0 |   if (CR != conversionOK) { | 
| 125 | 0 |     Out.clear(); | 
| 126 | 0 |     return false; | 
| 127 | 0 |   } | 
| 128 | 0 |  | 
| 129 | 0 |   Out.resize(reinterpret_cast<char *>(Dst) - &Out[0]); | 
| 130 | 0 |   Out.push_back(0); | 
| 131 | 0 |   Out.pop_back(); | 
| 132 | 0 |   return true; | 
| 133 | 0 | } | 
| 134 |  |  | 
| 135 |  | bool convertUTF16ToUTF8String(ArrayRef<UTF16> Src, std::string &Out) | 
| 136 | 0 | { | 
| 137 | 0 |   return convertUTF16ToUTF8String( | 
| 138 | 0 |       llvm::ArrayRef<char>(reinterpret_cast<const char *>(Src.data()), | 
| 139 | 0 |       Src.size() * sizeof(UTF16)), Out); | 
| 140 | 0 | } | 
| 141 |  |  | 
| 142 |  | bool convertUTF8ToUTF16String(StringRef SrcUTF8, | 
| 143 | 0 |                               SmallVectorImpl<UTF16> &DstUTF16) { | 
| 144 | 0 |   assert(DstUTF16.empty()); | 
| 145 | 0 | 
 | 
| 146 | 0 |   // Avoid OOB by returning early on empty input. | 
| 147 | 0 |   if (SrcUTF8.empty()) { | 
| 148 | 0 |     DstUTF16.push_back(0); | 
| 149 | 0 |     DstUTF16.pop_back(); | 
| 150 | 0 |     return true; | 
| 151 | 0 |   } | 
| 152 | 0 |  | 
| 153 | 0 |   const UTF8 *Src = reinterpret_cast<const UTF8 *>(SrcUTF8.begin()); | 
| 154 | 0 |   const UTF8 *SrcEnd = reinterpret_cast<const UTF8 *>(SrcUTF8.end()); | 
| 155 | 0 | 
 | 
| 156 | 0 |   // Allocate the same number of UTF-16 code units as UTF-8 code units. Encoding | 
| 157 | 0 |   // as UTF-16 should always require the same amount or less code units than the | 
| 158 | 0 |   // UTF-8 encoding.  Allocate one extra byte for the null terminator though, | 
| 159 | 0 |   // so that someone calling DstUTF16.data() gets a null terminated string. | 
| 160 | 0 |   // We resize down later so we don't have to worry that this over allocates. | 
| 161 | 0 |   DstUTF16.resize(SrcUTF8.size()+1); | 
| 162 | 0 |   UTF16 *Dst = &DstUTF16[0]; | 
| 163 | 0 |   UTF16 *DstEnd = Dst + DstUTF16.size(); | 
| 164 | 0 | 
 | 
| 165 | 0 |   ConversionResult CR = | 
| 166 | 0 |       ConvertUTF8toUTF16(&Src, SrcEnd, &Dst, DstEnd, strictConversion); | 
| 167 | 0 |   assert(CR != targetExhausted); | 
| 168 | 0 | 
 | 
| 169 | 0 |   if (CR != conversionOK) { | 
| 170 | 0 |     DstUTF16.clear(); | 
| 171 | 0 |     return false; | 
| 172 | 0 |   } | 
| 173 | 0 |  | 
| 174 | 0 |   DstUTF16.resize(Dst - &DstUTF16[0]); | 
| 175 | 0 |   DstUTF16.push_back(0); | 
| 176 | 0 |   DstUTF16.pop_back(); | 
| 177 | 0 |   return true; | 
| 178 | 0 | } | 
| 179 |  |  | 
| 180 |  | static_assert(sizeof(wchar_t) == 1 || sizeof(wchar_t) == 2 || | 
| 181 |  |                   sizeof(wchar_t) == 4, | 
| 182 |  |               "Expected wchar_t to be 1, 2, or 4 bytes"); | 
| 183 |  |  | 
| 184 |  | template <typename TResult> | 
| 185 |  | static inline bool ConvertUTF8toWideInternal(llvm::StringRef Source, | 
| 186 | 0 |                                              TResult &Result) { | 
| 187 | 0 |   // Even in the case of UTF-16, the number of bytes in a UTF-8 string is | 
| 188 | 0 |   // at least as large as the number of elements in the resulting wide | 
| 189 | 0 |   // string, because surrogate pairs take at least 4 bytes in UTF-8. | 
| 190 | 0 |   Result.resize(Source.size() + 1); | 
| 191 | 0 |   char *ResultPtr = reinterpret_cast<char *>(&Result[0]); | 
| 192 | 0 |   const UTF8 *ErrorPtr; | 
| 193 | 0 |   if (!ConvertUTF8toWide(sizeof(wchar_t), Source, ResultPtr, ErrorPtr)) { | 
| 194 | 0 |     Result.clear(); | 
| 195 | 0 |     return false; | 
| 196 | 0 |   } | 
| 197 | 0 |   Result.resize(reinterpret_cast<wchar_t *>(ResultPtr) - &Result[0]); | 
| 198 | 0 |   return true; | 
| 199 | 0 | } | 
| 200 |  |  | 
| 201 | 0 | bool ConvertUTF8toWide(llvm::StringRef Source, std::wstring &Result) { | 
| 202 | 0 |   return ConvertUTF8toWideInternal(Source, Result); | 
| 203 | 0 | } | 
| 204 |  |  | 
| 205 | 0 | bool ConvertUTF8toWide(const char *Source, std::wstring &Result) { | 
| 206 | 0 |   if (!Source) { | 
| 207 | 0 |     Result.clear(); | 
| 208 | 0 |     return true; | 
| 209 | 0 |   } | 
| 210 | 0 |   return ConvertUTF8toWide(llvm::StringRef(Source), Result); | 
| 211 | 0 | } | 
| 212 |  |  | 
| 213 | 0 | bool convertWideToUTF8(const std::wstring &Source, std::string &Result) { | 
| 214 | 0 |   if (sizeof(wchar_t) == 1) { | 
| 215 | 0 |     const UTF8 *Start = reinterpret_cast<const UTF8 *>(Source.data()); | 
| 216 | 0 |     const UTF8 *End = | 
| 217 | 0 |         reinterpret_cast<const UTF8 *>(Source.data() + Source.size()); | 
| 218 | 0 |     if (!isLegalUTF8String(&Start, End)) | 
| 219 | 0 |       return false; | 
| 220 | 0 |     Result.resize(Source.size()); | 
| 221 | 0 |     memcpy(&Result[0], Source.data(), Source.size()); | 
| 222 | 0 |     return true; | 
| 223 | 0 |   } else if (sizeof(wchar_t) == 2) { | 
| 224 | 0 |     return convertUTF16ToUTF8String( | 
| 225 | 0 |         llvm::ArrayRef<UTF16>(reinterpret_cast<const UTF16 *>(Source.data()), | 
| 226 | 0 |                               Source.size()), | 
| 227 | 0 |         Result); | 
| 228 | 0 |   } else if (sizeof(wchar_t) == 4) { | 
| 229 | 0 |     const UTF32 *Start = reinterpret_cast<const UTF32 *>(Source.data()); | 
| 230 | 0 |     const UTF32 *End = | 
| 231 | 0 |         reinterpret_cast<const UTF32 *>(Source.data() + Source.size()); | 
| 232 | 0 |     Result.resize(UNI_MAX_UTF8_BYTES_PER_CODE_POINT * Source.size()); | 
| 233 | 0 |     UTF8 *ResultPtr = reinterpret_cast<UTF8 *>(&Result[0]); | 
| 234 | 0 |     UTF8 *ResultEnd = reinterpret_cast<UTF8 *>(&Result[0] + Result.size()); | 
| 235 | 0 |     if (ConvertUTF32toUTF8(&Start, End, &ResultPtr, ResultEnd, | 
| 236 | 0 |                            strictConversion) == conversionOK) { | 
| 237 | 0 |       Result.resize(reinterpret_cast<char *>(ResultPtr) - &Result[0]); | 
| 238 | 0 |       return true; | 
| 239 | 0 |     } else { | 
| 240 | 0 |       Result.clear(); | 
| 241 | 0 |       return false; | 
| 242 | 0 |     } | 
| 243 | 0 |   } else { | 
| 244 | 0 |     llvm_unreachable( | 
| 245 | 0 |         "Control should never reach this point; see static_assert further up"); | 
| 246 | 0 |   } | 
| 247 | 0 | } | 
| 248 |  |  | 
| 249 |  | } // end namespace llvm | 
| 250 |  |  |