Internal change

PiperOrigin-RevId: 598901559

sharing/internal/base/BUILD

@@ -0,0 +1,67 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+cc_library(
+    name = "utf_utils",
+    srcs = [
+        "utf_string_conversions.cc",
+    ],
+    hdrs = [
+        "utf_string_configuration.h",
+        "utf_string_conversions.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//sharing/internal/public:logging",
+        "//third_party/icu_utf:icu_utf_assistant",
+    ],
+)
+
+cc_library(
+    name = "base",
+    srcs = [
+        "encode.cc",
+        "mime.cc",
+    ],
+    hdrs = [
+        "encode.h",
+        "mime.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:span",
+    ],
+)
+
+cc_test(
+    name = "base_test",
+    size = "small",
+    timeout = "short",
+    srcs = [
+        "encode_test.cc",
+        "utf_string_conversions_test.cc",
+    ],
+    shard_count = 8,
+    deps = [
+        ":base",
+        ":utf_utils",
+        "//internal/platform/implementation/g3",  # fixdeps: keep
+        "@com_github_protobuf_matchers//protobuf-matchers",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

sharing/internal/base/encode.cc

@@ -0,0 +1,42 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "sharing/internal/base/encode.h"
+
+#include <stdint.h>
+
+#include <iomanip>
+#include <ios>
+#include <sstream>
+#include <string>
+
+#include "absl/types/span.h"
+
+namespace nearby {
+namespace utils {
+
+// Returns uppercase string.
+std::string HexEncode(absl::Span<const uint8_t> data) {
+  std::ostringstream stream;
+  stream << std::hex << std::setfill('0') << std::uppercase;
+
+  for (uint8_t val : data) {
+    stream << std::setw(2) << static_cast<int>(val);
+  }
+
+  return stream.str();
+}
+
+}  // namespace utils
+}  // namespace nearby

sharing/internal/base/encode.h

@@ -0,0 +1,32 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_ENCODE_H_
+#define THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_ENCODE_H_
+
+#include <stdint.h>
+
+#include <string>
+
+#include "absl/types/span.h"
+
+namespace nearby {
+namespace utils {
+
+std::string HexEncode(absl::Span<const uint8_t> data);
+
+}  // namespace utils
+}  // namespace nearby
+
+#endif  // THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_ENCODE_H_

sharing/internal/base/encode_test.cc

@@ -0,0 +1,42 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "sharing/internal/base/encode.h"
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "gtest/gtest.h"
+
+namespace nearby {
+namespace utils {
+namespace {
+
+TEST(HexEncode, HexEncodeNormal) {
+  std::vector<uint8_t> data = {0x04, 0x3f, 0xf7, 0xe6, 0xf8, 0xc7, 0x0c};
+  EXPECT_EQ(HexEncode(data), "043FF7E6F8C70C");
+}
+
+TEST(HexEncode, HexEncodeEmpty) {
+  EXPECT_EQ(HexEncode(std::vector<uint8_t>({})), "");
+}
+
+TEST(HexEncode, HexEncodeWithPadding) {
+  EXPECT_EQ(HexEncode(std::vector<uint8_t>({0, 0, 0, 0})), "00000000");
+}
+
+}  // namespace
+}  // namespace utils
+}  // namespace nearby

sharing/internal/base/mime.h

@@ -0,0 +1,30 @@
+// Copyright 2022 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_MIME_H_
+#define THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_MIME_H_
+
+#include <string>
+
+#include "absl/strings/string_view.h"
+
+namespace nearby {
+namespace utils {
+
+std::string GetWellKnownMimeTypeFromExtension(absl::string_view extension);
+
+}  // namespace utils
+}  // namespace nearby
+
+#endif  // THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_MIME_H_

sharing/internal/base/utf_string_configuration.h

@@ -0,0 +1,120 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_UTF_STRING_CONFIGURATION_H_
+#define THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_UTF_STRING_CONFIGURATION_H_
+
+// A set of macros to use for platform detection.
+#if defined(__native_client__)
+// __native_client__ must be first, so that other OS_ defines are not set.
+#define OS_NACL 1
+#define OS_NACL_SFI
+#elif defined(ANDROID)
+#define OS_ANDROID 1
+#elif defined(__APPLE__)
+// Only include TargetConditionals after testing ANDROID as some Android builds
+// on the Mac have this header available and it's not needed unless the target
+// is really an Apple platform.
+#include <TargetConditionals.h>
+#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE
+#define OS_IOS 1
+#else
+#define OS_MAC 1
+#endif  // defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE
+#elif defined(__linux__)
+#if !defined(OS_CHROMEOS)
+// Do not define OS_LINUX on Chrome OS build.
+// The OS_CHROMEOS macro is defined in GN.
+#define OS_LINUX 1
+#endif  // !defined(OS_CHROMEOS)
+#if defined(__GLIBC__) && !defined(__UCLIBC__)
+// We really are using glibc, not uClibc pretending to be glibc.
+#define LIBC_GLIBC 1
+#endif
+#elif defined(_WIN32)
+#define OS_WIN 1
+#elif defined(__Fuchsia__)
+#define OS_FUCHSIA 1
+#elif defined(__FreeBSD__)
+#define OS_FREEBSD 1
+#elif defined(__NetBSD__)
+#define OS_NETBSD 1
+#elif defined(__OpenBSD__)
+#define OS_OPENBSD 1
+#elif defined(__sun)
+#define OS_SOLARIS 1
+#elif defined(__QNXNTO__)
+#define OS_QNX 1
+#elif defined(_AIX)
+#define OS_AIX 1
+#elif defined(__asmjs__) || defined(__wasm__)
+#define OS_ASMJS 1
+#elif defined(__MVS__)
+#define OS_ZOS 1
+#else
+#error Please add support for your platform in build/build_config.h
+#endif
+// NOTE: Adding a new port? Please follow
+// https://chromium.googlesource.com/chromium/src/+/main/docs/new_port_policy.md
+
+#if defined(OS_MAC) || defined(OS_IOS)
+#define OS_APPLE 1
+#endif
+
+// For access to standard BSD features, use OS_BSD instead of a
+// more specific macro.
+#if defined(OS_FREEBSD) || defined(OS_NETBSD) || defined(OS_OPENBSD)
+#define OS_BSD 1
+#endif
+
+// For access to standard POSIX features, use OS_POSIX instead of a
+// more specific macro.
+#if defined(OS_AIX) || defined(OS_ANDROID) || defined(OS_ASMJS) ||  \
+    defined(OS_FREEBSD) || defined(OS_IOS) || defined(OS_LINUX) ||  \
+    defined(OS_CHROMEOS) || defined(OS_MAC) || defined(OS_NACL) ||  \
+    defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_QNX) || \
+    defined(OS_SOLARIS) || defined(OS_ZOS)
+#define OS_POSIX 1
+#endif
+
+// Compiler detection. Note: clang masquerades as GCC on POSIX and as MSVC on
+// Windows.
+#if defined(__GNUC__)
+#define COMPILER_GCC 1
+#elif defined(_MSC_VER)
+#define COMPILER_MSVC 1
+#else
+#error Please add support for your compiler in build/build_config.h
+#endif
+
+// Type detection for wchar_t.
+#if defined(OS_WIN)
+#define WCHAR_T_IS_UTF16
+#elif defined(OS_FUCHSIA)
+#define WCHAR_T_IS_UTF32
+#elif defined(OS_POSIX) && defined(COMPILER_GCC) && defined(__WCHAR_MAX__) && \
+    (__WCHAR_MAX__ == 0x7fffffff || __WCHAR_MAX__ == 0xffffffff)
+#define WCHAR_T_IS_UTF32
+#elif defined(OS_POSIX) && defined(COMPILER_GCC) && defined(__WCHAR_MAX__) && \
+    (__WCHAR_MAX__ == 0x7fff || __WCHAR_MAX__ == 0xffff)
+// On Posix, we'll detect short wchar_t, but projects aren't guaranteed to
+// compile in this mode (in particular, Chrome doesn't). This is intended for
+// other projects using base who manage their own dependencies and make sure
+// short wchar works for them.
+#define WCHAR_T_IS_UTF16
+#else
+#error Please add support for your compiler in build/build_config.h
+#endif
+
+#endif  // THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_UTF_STRING_CONFIGURATION_H_

sharing/internal/base/utf_string_conversions.cc

@@ -0,0 +1,510 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "sharing/internal/base/utf_string_conversions.h"
+
+#include <limits.h>
+#include <stdint.h>
+
+#include <cstddef>
+#include <string>
+#include <string_view>
+#include <type_traits>
+
+#include "third_party/icu_utf/icu_utf.h"
+#include "sharing/internal/public/logging.h"
+
+namespace nearby {
+namespace utils {
+namespace {
+
+using MachineWord = uintptr_t;
+
+constexpr int32_t kErrorCodePoint = 0xFFFD;
+
+inline bool IsMachineWordAligned(const void* pointer) {
+  return !(reinterpret_cast<MachineWord>(pointer) & (sizeof(MachineWord) - 1));
+}
+
+template <class Char>
+bool DoIsStringAscii(const Char* characters, size_t length) {
+  // Bitmasks to detect non-ASCII characters for character sizes of 8, 16 and 32
+  // bits.
+  constexpr MachineWord NonASCIIMasks[] = {
+      0, MachineWord(0x8080808080808080ULL), MachineWord(0xFF80FF80FF80FF80ULL),
+      0, MachineWord(0xFFFFFF80FFFFFF80ULL),
+  };
+
+  if (!length) return true;
+  constexpr MachineWord non_ascii_bit_mask = NonASCIIMasks[sizeof(Char)];
+  static_assert(non_ascii_bit_mask, "Error: Invalid Mask");
+  MachineWord all_char_bits = 0;
+  const Char* end = characters + length;
+
+  // Prologue: align the input.
+  while (!IsMachineWordAligned(characters) && characters < end)
+    all_char_bits |= *characters++;
+  if (all_char_bits & non_ascii_bit_mask) return false;
+
+  // Compare the values of CPU word size.
+  constexpr size_t chars_per_word = sizeof(MachineWord) / sizeof(Char);
+  constexpr int batch_count = 16;
+  while (characters <= end - batch_count * chars_per_word) {
+    all_char_bits = 0;
+    for (int i = 0; i < batch_count; ++i) {
+      all_char_bits |= *(reinterpret_cast<const MachineWord*>(characters));
+      characters += chars_per_word;
+    }
+    if (all_char_bits & non_ascii_bit_mask) return false;
+  }
+
+  // Process the remaining words.
+  all_char_bits = 0;
+  while (characters <= end - chars_per_word) {
+    all_char_bits |= *(reinterpret_cast<const MachineWord*>(characters));
+    characters += chars_per_word;
+  }
+
+  // Process the remaining bytes.
+  while (characters < end) all_char_bits |= *characters++;
+
+  return !(all_char_bits & non_ascii_bit_mask);
+}
+
+inline bool IsValidCharacter(uint32_t code_point) {
+  // Excludes non-characters (U+FDD0..U+FDEF, and all code points
+  // ending in 0xFFFE or 0xFFFF) from the set of valid code points.
+  // https://unicode.org/faq/private_use.html#nonchar1
+  return code_point < 0xD800u ||
+         (code_point >= 0xE000u && code_point < 0xFDD0u) ||
+         (code_point > 0xFDEFu && code_point <= 0x10FFFFu &&
+          (code_point & 0xFFFEu) != 0xFFFEu);
+}
+
+template <bool (*Validator)(uint32_t)>
+inline bool DoIsStringUtf8(std::string_view str) {
+  const char* src = str.data();
+  int32_t src_len = static_cast<int32_t>(str.length());
+  int32_t char_index = 0;
+
+  while (char_index < src_len) {
+    int32_t code_point;
+    CBU8_NEXT(src, char_index, src_len, code_point);
+    if (!Validator(code_point)) return false;
+  }
+  return true;
+}
+
+// Size coefficient ----------------------------------------------------------
+// The maximum number of codeunits in the destination encoding corresponding to
+// one codeunit in the source encoding.
+
+template <typename SrcChar, typename DestChar>
+struct SizeCoefficient {
+  static_assert(sizeof(SrcChar) < sizeof(DestChar),
+                "Default case: from a smaller encoding to the bigger one");
+
+  // ASCII symbols are encoded by one codeunit in all encodings.
+  static constexpr int value = 1;
+};
+
+template <>
+struct SizeCoefficient<char16_t, char> {
+  // One UTF-16 code unit corresponds to at most 3 code units in UTF-8.
+  static constexpr int value = 3;
+};
+
+#if defined(WCHAR_T_IS_UTF32)
+template <>
+struct SizeCoefficient<wchar_t, char> {
+  // UTF-8 uses at most 4 code units per character.
+  static constexpr int value = 4;
+};
+
+template <>
+struct SizeCoefficient<wchar_t, char16_t> {
+  // UTF-16 uses at most 2 code units per character.
+  static constexpr int value = 2;
+};
+#endif  // defined(WCHAR_T_IS_UTF32)
+
+template <typename SrcChar, typename DestChar>
+constexpr int size_coefficient_v =
+    SizeCoefficient<std::decay_t<SrcChar>, std::decay_t<DestChar>>::value;
+
+// UnicodeAppendUnsafe --------------------------------------------------------
+// Function overloads that write code_point to the output string. Output string
+// has to have enough space for the codepoint.
+
+// Convenience typedef that checks whether the passed in type is integral (i.e.
+// bool, char, int or their extended versions) and is of the correct size.
+template <typename Char, size_t N>
+using EnableIfBitsAre = std::enable_if_t<
+    std::is_integral<Char>::value && CHAR_BIT * sizeof(Char) == N, bool>;
+
+template <typename Char, EnableIfBitsAre<Char, 8> = true>
+void UnicodeAppendUnsafe(Char* out, int32_t* size, uint32_t code_point) {
+  CBU8_APPEND_UNSAFE(out, *size, code_point);
+}
+
+template <typename Char, EnableIfBitsAre<Char, 16> = true>
+void UnicodeAppendUnsafe(Char* out, int32_t* size, uint32_t code_point) {
+  CBU16_APPEND_UNSAFE(out, *size, code_point);
+}
+
+template <typename Char, EnableIfBitsAre<Char, 32> = true>
+void UnicodeAppendUnsafe(Char* out, int32_t* size, uint32_t code_point) {
+  out[(*size)++] = code_point;
+}
+
+// DoUtfConversion ------------------------------------------------------------
+// Main driver of UtfConversion specialized for different Src encodings.
+// dest has to have enough room for the converted text.
+
+template <typename DestChar>
+bool DoUtfConversion(const char* src, int32_t src_len, DestChar* dest,
+                     int32_t* dest_len) {
+  bool success = true;
+
+  for (int32_t i = 0; i < src_len;) {
+    int32_t code_point;
+    CBU8_NEXT(src, i, src_len, code_point);
+
+    if (!IsValidCodepoint(code_point)) {
+      success = false;
+      code_point = kErrorCodePoint;
+    }
+
+    UnicodeAppendUnsafe(dest, dest_len, code_point);
+  }
+
+  return success;
+}
+
+template <typename DestChar>
+bool DoUtfConversion(const char16_t* src, int32_t src_len, DestChar* dest,
+                     int32_t* dest_len) {
+  bool success = true;
+
+  auto ConvertSingleChar = [&success](char16_t in) -> int32_t {
+    if (!CBU16_IS_SINGLE(in) || !IsValidCodepoint(in)) {
+      success = false;
+      return kErrorCodePoint;
+    }
+    return in;
+  };
+
+  int32_t i = 0;
+
+  // Always have another symbol in order to avoid checking boundaries in the
+  // middle of the surrogate pair.
+  while (i < src_len - 1) {
+    int32_t code_point;
+
+    if (CBU16_IS_LEAD(src[i]) && CBU16_IS_TRAIL(src[i + 1])) {
+      code_point = CBU16_GET_SUPPLEMENTARY(src[i], src[i + 1]);
+      if (!IsValidCodepoint(code_point)) {
+        code_point = kErrorCodePoint;
+        success = false;
+      }
+      i += 2;
+    } else {
+      code_point = ConvertSingleChar(src[i]);
+      ++i;
+    }
+
+    UnicodeAppendUnsafe(dest, dest_len, code_point);
+  }
+
+  if (i < src_len)
+    UnicodeAppendUnsafe(dest, dest_len, ConvertSingleChar(src[i]));
+
+  return success;
+}
+
+#if defined(WCHAR_T_IS_UTF32)
+
+template <typename DestChar>
+bool DoUtfConversion(const wchar_t* src, int32_t src_len, DestChar* dest,
+                     int32_t* dest_len) {
+  bool success = true;
+
+  for (int32_t i = 0; i < src_len; ++i) {
+    int32_t code_point = src[i];
+
+    if (!IsValidCodepoint(code_point)) {
+      success = false;
+      code_point = kErrorCodePoint;
+    }
+
+    UnicodeAppendUnsafe(dest, dest_len, code_point);
+  }
+
+  return success;
+}
+
+#endif  // defined(WCHAR_T_IS_UTF32)
+
+// UtfConversion --------------------------------------------------------------
+// Function template for generating all UTF conversions.
+
+template <typename InputString, typename DestString>
+bool UtfConversion(const InputString& src_str, DestString* dest_str) {
+  if (IsStringAscii(src_str)) {
+    dest_str->assign(src_str.begin(), src_str.end());
+    return true;
+  }
+
+  dest_str->resize(src_str.length() *
+                   size_coefficient_v<typename InputString::value_type,
+                                      typename DestString::value_type>);
+
+  // Empty string is ASCII => it OK to call operator[].
+  auto* dest = &(*dest_str)[0];
+
+  // ICU requires 32 bit numbers.
+  int32_t src_len32 = static_cast<int32_t>(src_str.length());
+  int32_t dest_len32 = 0;
+
+  bool res = DoUtfConversion(src_str.data(), src_len32, dest, &dest_len32);
+
+  dest_str->resize(dest_len32);
+  dest_str->shrink_to_fit();
+
+  return res;
+}
+
+#if defined(WCHAR_T_IS_UTF16)
+inline const char16_t* as_u16cstr(const wchar_t* str) {
+  return reinterpret_cast<const char16_t*>(str);
+}
+
+inline const char16_t* as_u16cstr(std::wstring_view str) {
+  return reinterpret_cast<const char16_t*>(str.data());
+}
+#endif
+
+}  // namespace
+
+// UTF16 <-> UTF8 --------------------------------------------------------------
+
+bool Utf8ToUtf16(const char* src, size_t src_len, std::u16string* output) {
+  return UtfConversion(std::string_view(src, src_len), output);
+}
+
+std::u16string Utf8ToUtf16(std::string_view utf8) {
+  std::u16string ret;
+  // Ignore the success flag of this call, it will do the best it can for
+  // invalid input, which is what we want here.
+  Utf8ToUtf16(utf8.data(), utf8.size(), &ret);
+  return ret;
+}
+
+bool Utf16ToUtf8(const char16_t* src, size_t src_len, std::string* output) {
+  return UtfConversion(std::u16string_view(src, src_len), output);
+}
+
+std::string Utf16ToUtf8(std::u16string_view utf16) {
+  std::string ret;
+  // Ignore the success flag of this call, it will do the best it can for
+  // invalid input, which is what we want here.
+  Utf16ToUtf8(utf16.data(), utf16.length(), &ret);
+  return ret;
+}
+
+// UTF-16 <-> Wide -------------------------------------------------------------
+
+#if defined(WCHAR_T_IS_UTF16)
+// When wide == UTF-16 the conversions are a NOP.
+
+bool WideToUtf16(const wchar_t* src, size_t src_len, std::u16string* output) {
+  output->assign(src, src + src_len);
+  return true;
+}
+
+std::u16string WideToUtf16(std::wstring_view wide) {
+  return std::u16string(wide.begin(), wide.end());
+}
+
+bool Utf16ToWide(const char16_t* src, size_t src_len, std::wstring* output) {
+  output->assign(src, src + src_len);
+  return true;
+}
+
+std::wstring Utf16ToWide(std::u16string_view utf16) {
+  return std::wstring(utf16.begin(), utf16.end());
+}
+
+#elif defined(WCHAR_T_IS_UTF32)
+
+bool WideToUtf16(const wchar_t* src, size_t src_len, std::u16string* output) {
+  return UtfConversion(std::wstring_view(src, src_len), output);
+}
+
+std::u16string WideToUtf16(std::wstring_view wide) {
+  std::u16string ret;
+  // Ignore the success flag of this call, it will do the best it can for
+  // invalid input, which is what we want here.
+  WideToUtf16(wide.data(), wide.length(), &ret);
+  return ret;
+}
+
+bool Utf16ToWide(const char16_t* src, size_t src_len, std::wstring* output) {
+  return UtfConversion(std::u16string_view(src, src_len), output);
+}
+
+std::wstring Utf16ToWide(std::u16string_view utf16) {
+  std::wstring ret;
+  // Ignore the success flag of this call, it will do the best it can for
+  // invalid input, which is what we want here.
+  Utf16ToWide(utf16.data(), utf16.length(), &ret);
+  return ret;
+}
+
+#endif  // defined(WCHAR_T_IS_UTF32)
+
+// UTF-8 <-> Wide --------------------------------------------------------------
+
+// UTF8ToWide is the same code, regardless of whether wide is 16 or 32 bits
+
+bool Utf8ToWide(const char* src, size_t src_len, std::wstring* output) {
+  return UtfConversion(std::string_view(src, src_len), output);
+}
+
+std::wstring Utf8ToWide(std::string_view utf8) {
+  std::wstring ret;
+  // Ignore the success flag of this call, it will do the best it can for
+  // invalid input, which is what we want here.
+  Utf8ToWide(utf8.data(), utf8.length(), &ret);
+  return ret;
+}
+
+#if defined(WCHAR_T_IS_UTF16)
+// Easy case since we can use the "utf" versions we already wrote above.
+
+bool WideToUtf8(const wchar_t* src, size_t src_len, std::string* output) {
+  return Utf16ToUtf8(as_u16cstr(src), src_len, output);
+}
+
+std::string WideToUtf8(std::wstring_view wide) {
+  return Utf16ToUtf8(std::u16string_view(as_u16cstr(wide), wide.size()));
+}
+
+#elif defined(WCHAR_T_IS_UTF32)
+
+bool WideToUtf8(const wchar_t* src, size_t src_len, std::string* output) {
+  return UtfConversion(std::wstring_view(src, src_len), output);
+}
+
+std::string WideToUtf8(std::wstring_view wide) {
+  std::string ret;
+  // Ignore the success flag of this call, it will do the best it can for
+  // invalid input, which is what we want here.
+  WideToUtf8(wide.data(), wide.length(), &ret);
+  return ret;
+}
+
+#endif  // defined(WCHAR_T_IS_UTF32)
+
+std::u16string AsciiToUtf16(std::string_view ascii) {
+  NL_DCHECK(IsStringAscii(ascii));
+  return std::u16string(ascii.begin(), ascii.end());
+}
+
+std::string Utf16ToAscii(std::u16string_view utf16) {
+  NL_DCHECK(IsStringAscii(utf16));
+  return std::string(utf16.begin(), utf16.end());
+}
+
+#if defined(WCHAR_T_IS_UTF16)
+std::wstring AsciiToWide(std::string_view ascii) {
+  NL_DCHECK(IsStringAscii(ascii));
+  return std::wstring(ascii.begin(), ascii.end());
+}
+
+std::string WideToAscii(std::string_view wide) {
+  NL_DCHECK(IsStringAscii(wide));
+  return std::string(wide.begin(), wide.end());
+}
+#endif  // defined(WCHAR_T_IS_UTF16)
+
+bool IsStringAscii(std::string_view str) {
+  return DoIsStringAscii(str.data(), str.length());
+}
+
+bool IsStringAscii(std::u16string_view str) {
+  return DoIsStringAscii(str.data(), str.length());
+}
+
+bool IsStringUtf8(std::string_view str) {
+  return DoIsStringUtf8<IsValidCharacter>(str);
+}
+
+bool IsStringAscii(std::wstring_view str) {
+  return DoIsStringAscii(str.data(), str.length());
+}
+
+bool IsValidCodepoint(uint32_t code_point) {
+  // Excludes code points that are not Unicode scalar values, i.e.
+  // surrogate code points ([0xD800, 0xDFFF]). Additionally, excludes
+  // code points larger than 0x10FFFF (the highest codepoint allowed).
+  // Non-characters and unassigned code points are allowed.
+  // https://unicode.org/glossary/#unicode_scalar_value
+  return code_point < 0xD800u ||
+         (code_point >= 0xE000u && code_point <= 0x10FFFFu);
+}
+
+void TruncateUtf8ToByteSize(const std::string& input, size_t byte_size,
+                            std::string* output) {
+  NL_DCHECK(output);
+  if (byte_size > input.length()) {
+    *output = input;
+    return;
+  }
+
+  // Note: This cast is necessary because CBU8_NEXT uses int32_ts.
+  int32_t truncation_length = static_cast<int32_t>(byte_size);
+  int32_t char_index = truncation_length - 1;
+  const char* data = input.data();
+
+  // Using CBU8, we will move backwards from the truncation point
+  // to the beginning of the string looking for a valid UTF8
+  // character.  Once a full UTF8 character is found, we will
+  // truncate the string to the end of that character.
+  while (char_index >= 0) {
+    int32_t prev = char_index;
+    int32_t code_point = 0;
+    CBU8_NEXT(data, char_index, truncation_length, code_point);
+    if (!IsValidCharacter(code_point) || !IsValidCodepoint(code_point)) {
+      char_index = prev - 1;
+    } else {
+      break;
+    }
+  }
+
+  if (char_index >= 0)
+    *output = input.substr(0, char_index);
+  else
+    output->clear();
+}
+
+std::string ToString(const char* str) {
+  if (str == nullptr) {
+    return "";
+  }
+  return std::string(str);
+}
+
+}  // namespace utils
+}  // namespace nearby

sharing/internal/base/utf_string_conversions.h

@@ -0,0 +1,117 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_UTF_STRING_CONVERSIONS_H_
+#define THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_UTF_STRING_CONVERSIONS_H_
+
+#include <stddef.h>
+
+#include <cstdint>
+#include <string>
+#include <string_view>
+
+#include "sharing/internal/base/utf_string_configuration.h"
+
+namespace nearby {
+namespace utils {
+
+// These convert between UTF-8, -16, and -32 strings. They are potentially slow,
+// so avoid unnecessary conversions. The low-level versions return a boolean
+// indicating whether the conversion was 100% valid. In this case, it will still
+// do the best it can and put the result in the output buffer. The versions that
+// return strings ignore this error and just return the best conversion
+// possible.
+bool WideToUtf8(const wchar_t* src, size_t src_len, std::string* output);
+std::string WideToUtf8(std::wstring_view wide);
+bool Utf8ToWide(const char* src, size_t src_len, std::wstring* output);
+std::wstring Utf8ToWide(std::string_view utf8);
+
+bool WideToUtf16(const wchar_t* src, size_t src_len, std::u16string* output);
+std::u16string WideToUtf16(std::wstring_view wide);
+bool Utf16ToWide(const char16_t* src, size_t src_len, std::wstring* output);
+std::wstring Utf16ToWide(std::u16string_view utf16);
+
+bool Utf8ToUtf16(const char* src, size_t src_len, std::u16string* output);
+std::u16string Utf8ToUtf16(std::string_view utf8);
+bool Utf16ToUtf8(const char16_t* src, size_t src_len, std::string* output);
+std::string Utf16ToUtf8(std::u16string_view utf16);
+
+// This converts an ASCII string, typically a hard coded constant, to a UTF16
+// string.
+std::u16string AsciiToUtf16(std::string_view ascii);
+
+// Converts to 7-bit ASCII by truncating. The result must be known to be ASCII
+// beforehand.
+std::string Utf16ToAscii(std::u16string_view utf16);
+
+bool IsStringAscii(std::string_view str);
+bool IsStringAscii(std::u16string_view str);
+bool IsStringAscii(std::wstring_view str);
+bool IsStringUtf8(std::string_view str);
+
+inline bool IsValidCodepoint(uint32_t code_point);
+
+void TruncateUtf8ToByteSize(const std::string& input, size_t byte_size,
+                            std::string* output);
+
+#if defined(WCHAR_T_IS_UTF16)
+// This converts an ASCII string, typically a hard coded constant, to a wide
+// string.
+std::wstring AsciiToWide(std::string_view ascii);
+
+// Converts to 7-bit ASCII by truncating. The result must be known to be ASCII
+// beforehand.
+std::string WideToAscii(std::wstring_view wide);
+#endif  // defined(WCHAR_T_IS_UTF16)
+
+// The conversion functions in this file should not be used to convert string
+// literals. Instead, the corresponding prefixes (e.g. u"" for UTF16 or L"" for
+// Wide) should be used. Deleting the overloads here catches these cases at
+// compile time.
+template <size_t N>
+std::u16string WideToUtf16(const wchar_t (&str)[N]) {
+  static_assert(N == 0, "Error: Use the u\"...\" prefix instead.");
+  return std::u16string();
+}
+
+template <size_t N>
+std::u16string Utf8ToUtf16(const char (&str)[N]) {
+  static_assert(N == 0, "Error: Use the u\"...\" prefix instead.");
+  return std::u16string();
+}
+
+template <size_t N>
+std::u16string AsciiToUtf16(const char (&str)[N]) {
+  static_assert(N == 0, "Error: Use the u\"...\" prefix instead.");
+  return std::u16string();
+}
+
+// Mutable character arrays are usually only populated during runtime. Continue
+// to allow this conversion.
+template <size_t N>
+std::u16string AsciiToUtf16(char (&str)[N]) {
+  return AsciiToUtf16(std::string_view(str));
+}
+
+template <typename T, size_t N>
+constexpr size_t size(const T (&array)[N]) noexcept {
+  return N;
+}
+
+std::string ToString(const char* str);
+
+}  // namespace utils
+}  // namespace nearby
+
+#endif  // THIRD_PARTY_NEARBY_SHARING_INTERNAL_BASE_UTF_STRING_CONVERSIONS_H_

sharing/internal/base/utf_string_conversions_test.cc

@@ -0,0 +1,375 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "sharing/internal/base/utf_string_conversions.h"
+
+#include <cstring>
+#include <cwchar>
+#include <sstream>
+#include <string>
+
+#include "gtest/gtest.h"
+
+namespace nearby {
+namespace utils {
+namespace {
+
+const wchar_t* const kConvertRoundtripCases[] = {
+    L"Google Video",
+    // "网页 图片 资讯更多 »"
+    L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb",
+    //  "Παγκόσμιος Ιστός"
+    L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9"
+    L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2",
+    // "Поиск страниц на русском"
+    L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442"
+    L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430"
+    L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c",
+    // "전체서비스"
+    L"\xc804\xccb4\xc11c\xbe44\xc2a4",
+
+// Test characters that take more than 16 bits. This will depend on whether
+// wchar_t is 16 or 32 bits.
+#if defined(WCHAR_T_IS_UTF16)
+    L"\xd800\xdf00",
+    // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 :
+    // A,B,C,D,E)
+    L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44",
+#elif defined(WCHAR_T_IS_UTF32)
+    L"\x10300",
+    // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 :
+    // A,B,C,D,E)
+    L"\x11d40\x11d41\x11d42\x11d43\x11d44",
+#endif
+};
+
+// Helper used to test TruncateUtf8ToByteSize.
+bool Truncated(const std::string& input, const size_t byte_size,
+               std::string* output) {
+  size_t prev = input.length();
+  TruncateUtf8ToByteSize(input, byte_size, output);
+  return prev != output->length();
+}
+
+}  // namespace
+
+TEST(UtfStringConversionsTest, ConvertUtf8AndWide) {
+  // we round-trip all the wide strings through UTF-8 to make sure everything
+  // agrees on the conversion. This uses the stream operators to test them
+  // simultaneously.
+  for (auto* i : kConvertRoundtripCases) {
+    std::ostringstream utf8;
+    utf8 << WideToUtf8(i);
+    std::wostringstream wide;
+    wide << Utf8ToWide(utf8.str());
+
+    EXPECT_EQ(i, wide.str());
+  }
+}
+
+TEST(UtfStringConversionsTest, ConvertUtf8AndWideEmptyString) {
+  // An empty std::wstring should be converted to an empty std::string,
+  // and vice versa.
+  std::wstring wempty;
+  std::string empty;
+  EXPECT_EQ(empty, WideToUtf8(wempty));
+  EXPECT_EQ(wempty, Utf8ToWide(empty));
+}
+
+TEST(UtfStringConversionsTest, ConvertUtf8ToWide) {
+  struct Utf8ToWideCase {
+    const char* utf8;
+    const wchar_t* wide;
+    bool success;
+  } convert_cases[] = {
+    // Regular UTF-8 input.
+    {"\xe4\xbd\xa0\xe5\xa5\xbd", L"\x4f60\x597d", true},
+    // Non-character is passed through.
+    {"\xef\xbf\xbfHello", L"\xffffHello", true},
+    // Truncated UTF-8 sequence.
+    {"\xe4\xa0\xe5\xa5\xbd", L"\xfffd\x597d", false},
+    // Truncated off the end.
+    {"\xe5\xa5\xbd\xe4\xa0", L"\x597d\xfffd", false},
+    // Non-shortest-form UTF-8.
+    {"\xf0\x84\xbd\xa0\xe5\xa5\xbd", L"\xfffd\xfffd\xfffd\xfffd\x597d", false},
+    // This UTF-8 character is decoded to a UTF-16 surrogate, which is illegal.
+    {"\xed\xb0\x80", L"\xfffd\xfffd\xfffd", false},
+  // Non-BMP characters. The second is a non-character regarded as valid.
+  // The result will either be in UTF-16 or UTF-32.
+#if defined(WCHAR_T_IS_UTF16)
+    {"A\xF0\x90\x8C\x80z", L"A\xd800\xdf00z", true},
+    {"A\xF4\x8F\xBF\xBEz", L"A\xdbff\xdffez", true},
+#elif defined(WCHAR_T_IS_UTF32)
+    {"A\xF0\x90\x8C\x80z", L"A\x10300z", true},
+    {"A\xF4\x8F\xBF\xBEz", L"A\x10fffez", true},
+#endif
+  };
+
+  for (const auto& i : convert_cases) {
+    std::wstring converted;
+    EXPECT_EQ(i.success, Utf8ToWide(i.utf8, strlen(i.utf8), &converted));
+    std::wstring expected(i.wide);
+    EXPECT_EQ(expected, converted);
+  }
+
+  // Manually test an embedded NULL.
+  std::wstring converted;
+  EXPECT_TRUE(Utf8ToWide("\00Z\t", 3, &converted));
+  ASSERT_EQ(3U, converted.length());
+  EXPECT_EQ(static_cast<wchar_t>(0), converted[0]);
+  EXPECT_EQ('Z', converted[1]);
+  EXPECT_EQ('\t', converted[2]);
+
+  // Make sure that conversion replaces, not appends.
+  EXPECT_TRUE(Utf8ToWide("B", 1, &converted));
+  ASSERT_EQ(1U, converted.length());
+  EXPECT_EQ('B', converted[0]);
+}
+
+#if defined(WCHAR_T_IS_UTF16)
+// This test is only valid when wchar_t == UTF-16.
+TEST(UtfStringConversionsTest, ConvertUtf16ToUtf8) {
+  struct WideToUtf8Case {
+    const wchar_t* utf16;
+    const char* utf8;
+    bool success;
+  } convert_cases[] = {
+      // Regular UTF-16 input.
+      {L"\x4f60\x597d", "\xe4\xbd\xa0\xe5\xa5\xbd", true},
+      // Test a non-BMP character.
+      {L"\xd800\xdf00", "\xF0\x90\x8C\x80", true},
+      // Non-characters are passed through.
+      {L"\xffffHello", "\xEF\xBF\xBFHello", true},
+      {L"\xdbff\xdffeHello", "\xF4\x8F\xBF\xBEHello", true},
+      // The first character is a truncated UTF-16 character.
+      {L"\xd800\x597d", "\xef\xbf\xbd\xe5\xa5\xbd", false},
+      // Truncated at the end.
+      {L"\x597d\xd800", "\xe5\xa5\xbd\xef\xbf\xbd", false},
+  };
+
+  for (const auto& test : convert_cases) {
+    std::string converted;
+    EXPECT_EQ(test.success,
+              WideToUtf8(test.utf16, wcslen(test.utf16), &converted));
+    std::string expected(test.utf8);
+    EXPECT_EQ(expected, converted);
+  }
+}
+
+#elif defined(WCHAR_T_IS_UTF32)
+// This test is only valid when wchar_t == UTF-32.
+TEST(UtfStringConversionsTest, ConvertUtf32ToUtf8) {
+  struct WideToUtf8Case {
+    const wchar_t* utf32;
+    const char* utf8;
+    bool success;
+  } convert_cases[] = {
+      // Regular 16-bit input.
+      {L"\x4f60\x597d", "\xe4\xbd\xa0\xe5\xa5\xbd", true},
+      // Test a non-BMP character.
+      {L"A\x10300z", "A\xF0\x90\x8C\x80z", true},
+      // Non-characters are passed through.
+      {L"\xffffHello", "\xEF\xBF\xBFHello", true},
+      {L"\x10fffeHello", "\xF4\x8F\xBF\xBEHello", true},
+      // Invalid Unicode code points.
+      {L"\xfffffffHello", "\xEF\xBF\xBDHello", false},
+      // The first character is a truncated UTF-16 character.
+      {L"\xd800\x597d", "\xef\xbf\xbd\xe5\xa5\xbd", false},
+      {L"\xdc01Hello", "\xef\xbf\xbdHello", false},
+  };
+
+  for (const auto& test : convert_cases) {
+    std::string converted;
+    EXPECT_EQ(test.success,
+              WideToUtf8(test.utf32, wcslen(test.utf32), &converted));
+    std::string expected(test.utf8);
+    EXPECT_EQ(expected, converted);
+  }
+}
+#endif  // defined(WCHAR_T_IS_UTF32)
+
+TEST(UtfStringConversionsTest, TruncateUtf8ToByteSize) {
+  std::string output;
+
+  // Empty strings and invalid byte_size arguments
+  EXPECT_FALSE(Truncated(std::string(), 0, &output));
+  EXPECT_EQ(output, "");
+  EXPECT_TRUE(Truncated("\xe1\x80\xbf", 0, &output));
+  EXPECT_EQ(output, "");
+  EXPECT_FALSE(Truncated("\xe1\x80\xbf", static_cast<size_t>(-1), &output));
+  EXPECT_FALSE(Truncated("\xe1\x80\xbf", 4, &output));
+
+  // Testing the truncation of valid UTF8 correctly
+  EXPECT_TRUE(Truncated("abc", 2, &output));
+  EXPECT_EQ(output, "ab");
+  EXPECT_TRUE(Truncated("\xc2\x81\xc2\x81", 2, &output));
+  EXPECT_EQ(output.compare("\xc2\x81"), 0);
+  EXPECT_TRUE(Truncated("\xc2\x81\xc2\x81", 3, &output));
+  EXPECT_EQ(output.compare("\xc2\x81"), 0);
+  EXPECT_FALSE(Truncated("\xc2\x81\xc2\x81", 4, &output));
+  EXPECT_EQ(output.compare("\xc2\x81\xc2\x81"), 0);
+
+  {
+    const char array[] = "\x00\x00\xc2\x81\xc2\x81";
+    const std::string array_string(array, size(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\x00\x00\xc2\x81", 4)), 0);
+  }
+
+  {
+    const char array[] = "\x00\xc2\x81\xc2\x81";
+    const std::string array_string(array, size(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\x00\xc2\x81", 3)), 0);
+  }
+
+  // Testing invalid UTF8
+  EXPECT_TRUE(Truncated("\xed\xa0\x80\xed\xbf\xbf", 6, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xed\xa0\x8f", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xed\xbf\xbf", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // Testing invalid UTF8 mixed with valid UTF8
+  EXPECT_FALSE(Truncated("\xe1\x80\xbf", 3, &output));
+  EXPECT_EQ(output.compare("\xe1\x80\xbf"), 0);
+  EXPECT_FALSE(Truncated("\xf1\x80\xa0\xbf", 4, &output));
+  EXPECT_EQ(output.compare("\xf1\x80\xa0\xbf"), 0);
+  EXPECT_FALSE(Truncated("a\xc2\x81\xe1\x80\xbf\xf1\x80\xa0\xbf", 10, &output));
+  EXPECT_EQ(output.compare("a\xc2\x81\xe1\x80\xbf\xf1\x80\xa0\xbf"), 0);
+  EXPECT_TRUE(
+      Truncated("a\xc2\x81\xe1\x80\xbf\xf1"
+                "a"
+                "\x80\xa0",
+                10, &output));
+  EXPECT_EQ(output.compare("a\xc2\x81\xe1\x80\xbf\xf1"
+                           "a"),
+            0);
+  EXPECT_FALSE(
+      Truncated("\xef\xbb\xbf"
+                "abc",
+                6, &output));
+  EXPECT_EQ(output.compare("\xef\xbb\xbf"
+                           "abc"),
+            0);
+
+  // Overlong sequences
+  EXPECT_TRUE(Truncated("\xc0\x80", 2, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xc1\x80\xc1\x81", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xe0\x80\x80", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xe0\x82\x80", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xe0\x9f\xbf", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x80\x80\x8D", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x80\x82\x91", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x80\xa0\x80", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x8f\xbb\xbf", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf8\x80\x80\x80\xbf", 5, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xfc\x80\x80\x80\xa0\xa5", 6, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // Beyond U+10FFFF (the upper limit of Unicode codespace)
+  EXPECT_TRUE(Truncated("\xf4\x90\x80\x80", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf8\xa0\xbf\x80\xbf", 5, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xfc\x9c\xbf\x80\xbf\x80", 6, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // BOMs in UTF-16(BE|LE) and UTF-32(BE|LE)
+  EXPECT_TRUE(Truncated("\xfe\xff", 2, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xff\xfe", 2, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  {
+    const char array[] = "\x00\x00\xfe\xff";
+    const std::string array_string(array, size(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\x00\x00", 2)), 0);
+  }
+
+  // Variants on the previous test
+  {
+    const char array[] = "\xff\xfe\x00\x00";
+    const std::string array_string(array, 4);
+    EXPECT_FALSE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\xff\xfe\x00\x00", 4)), 0);
+  }
+  {
+    const char array[] = "\xff\x00\x00\xfe";
+    const std::string array_string(array, size(array));
+    EXPECT_TRUE(Truncated(array_string, 4, &output));
+    EXPECT_EQ(output.compare(std::string("\xff\x00\x00", 3)), 0);
+  }
+
+  // Non-characters : U+xxFFF[EF] where xx is 0x00 through 0x10 and <FDD0,FDEF>
+  EXPECT_TRUE(Truncated("\xef\xbf\xbe", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf0\x8f\xbf\xbe", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xf3\xbf\xbf\xbf", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xef\xb7\x90", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_TRUE(Truncated("\xef\xb7\xaf", 3, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // Strings in legacy encodings that are valid in UTF-8, but
+  // are invalid as UTF-8 in real data.
+  EXPECT_TRUE(Truncated("caf\xe9", 4, &output));
+  EXPECT_EQ(output.compare("caf"), 0);
+  EXPECT_TRUE(Truncated("\xb0\xa1\xb0\xa2", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+  EXPECT_FALSE(Truncated("\xa7\x41\xa6\x6e", 4, &output));
+  EXPECT_EQ(output.compare("\xa7\x41\xa6\x6e"), 0);
+  EXPECT_TRUE(Truncated("\xa7\x41\xa6\x6e\xd9\xee\xe4\xee", 7, &output));
+  EXPECT_EQ(output.compare("\xa7\x41\xa6\x6e"), 0);
+
+  // Testing using the same string as input and output.
+  EXPECT_FALSE(Truncated(output, 4, &output));
+  EXPECT_EQ(output.compare("\xa7\x41\xa6\x6e"), 0);
+  EXPECT_TRUE(Truncated(output, 3, &output));
+  EXPECT_EQ(output.compare("\xa7\x41"), 0);
+
+  // "abc" with U+201[CD] in windows-125[0-8]
+  EXPECT_TRUE(
+      Truncated("\x93"
+                "abc\x94",
+                5, &output));
+  EXPECT_EQ(output.compare("\x93"
+                           "abc"),
+            0);
+
+  // U+0639 U+064E U+0644 U+064E in ISO-8859-6
+  EXPECT_TRUE(Truncated("\xd9\xee\xe4\xee", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+
+  // U+03B3 U+03B5 U+03B9 U+03AC in ISO-8859-7
+  EXPECT_TRUE(Truncated("\xe3\xe5\xe9\xdC", 4, &output));
+  EXPECT_EQ(output.compare(""), 0);
+}
+
+}  // namespace utils
+}  // namespace nearby