Fix/align spark lower sigma semantics icu based

bolt/functions/lib/string/SparkLower.h

@@ -0,0 +1,229 @@
+/*
+ * Copyright (c) ByteDance Ltd. and/or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Copyright (c) ByteDance Ltd. and/or its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#pragma once
+
+#include <memory>
+
+#include <folly/CPortability.h>
+#include <folly/small_vector.h>
+#include <unicode/brkiter.h>
+#include <unicode/locid.h>
+#include <unicode/uchar.h>
+#include <unicode/unistr.h>
+
+#include "bolt/common/base/Exceptions.h"
+
+namespace bytedance::bolt::functions::stringCore::spark {
+
+FOLLY_ALWAYS_INLINE bool isJavaNumber(UChar32 codePoint) {
+  // Java groups all three Unicode number categories into its numeric word
+  // rules. This is also used to distinguish A1-Σ from AΣ-B below.
+  const auto category = u_charType(codePoint);
+  return category == U_DECIMAL_DIGIT_NUMBER || category == U_LETTER_NUMBER ||
+      category == U_OTHER_NUMBER;
+}
+
+FOLLY_ALWAYS_INLINE bool isJavaCased(UChar32 codePoint) {
+  // Java 11 uses Unicode 10, where Georgian Mkhedruli is an uncased Lo
+  // script. ICU 74 uses Unicode 15.1, where it is classified as Ll. Hence
+  // Java requires AΣა -> aςა, while ICU's Cased property produces aσა.
+  if ((codePoint >= 0x10D0 && codePoint <= 0x10FA) ||
+      (codePoint >= 0x10FD && codePoint <= 0x10FF)) {
+    return false;
+  }
+
+  const auto category = u_charType(codePoint);
+  return category == U_LOWERCASE_LETTER || category == U_UPPERCASE_LETTER ||
+      category == U_TITLECASE_LETTER ||
+      (codePoint >= 0x02B0 && codePoint <= 0x02B8) ||
+      (codePoint >= 0x02C0 && codePoint <= 0x02C1) ||
+      (codePoint >= 0x02E0 && codePoint <= 0x02E4) || codePoint == 0x0345 ||
+      codePoint == 0x037A || (codePoint >= 0x1D2C && codePoint <= 0x1D61) ||
+      (codePoint >= 0x2160 && codePoint <= 0x217F) ||
+      (codePoint >= 0x24B6 && codePoint <= 0x24E9);
+}
+
+FOLLY_ALWAYS_INLINE bool isJavaWordBoundary(
+    const icu::UnicodeString& input,
+    int32_t offset,
+    icu::BreakIterator& breakIterator) {
+  if (offset == 0 || offset == input.length()) {
+    return true;
+  }
+
+  const auto previousOffset = input.moveIndex32(offset, -1);
+  const auto previousCodePoint = input.char32At(previousOffset);
+  const auto nextCodePoint = input.char32At(offset);
+
+  // ICU 74 does not report these boundaries, but Java 11 treats both code
+  // points as word separators:
+  //   U+00B7: AΣ·B -> aς·b.
+  //   U+202F: AΣ\u202FB -> aς\u202Fb.
+  const auto forcesBoundary = [](UChar32 codePoint) {
+    return codePoint == 0x00B7 || codePoint == 0x202F;
+  };
+  if (forcesBoundary(previousCodePoint) || forcesBoundary(nextCodePoint)) {
+    return true;
+  }
+
+  // A hyphen joins Java words in AΣ-B -> aσ-b, but a hyphen following a
+  // number does not connect that number to the word on its right:
+  // A1-Σ -> a1-σ.
+  if (previousCodePoint == '-' && previousOffset > 0) {
+    const auto beforeHyphenOffset = input.moveIndex32(previousOffset, -1);
+    if (isJavaNumber(input.char32At(beforeHyphenOffset))) {
+      return true;
+    }
+  }
+
+  if (!breakIterator.isBoundary(offset)) {
+    return false;
+  }
+
+  // Remove boundaries that ICU 74 reports but Java 11 bridges:
+  //   Number: AΣ²B -> aσ²b.
+  //   Format: AΣ\u200BB -> aσ\u200Bb.
+  //   OtherLetter: AΣ가B -> aσ가b and AΣ㐀B -> aσ㐀b.
+  //   Hyphen: AΣ-B -> aσ-b (subject to the numeric override above).
+  const auto bridgesBoundary = [](UChar32 codePoint) {
+    const auto category = u_charType(codePoint);
+    return codePoint == '"' || codePoint == '-' ||
+        category == U_DECIMAL_DIGIT_NUMBER || category == U_LETTER_NUMBER ||
+        category == U_OTHER_NUMBER || category == U_FORMAT_CHAR ||
+        category == U_OTHER_LETTER;
+  };
+  return !bridgesBoundary(previousCodePoint) && !bridgesBoundary(nextCodePoint);
+}
+
+FOLLY_ALWAYS_INLINE bool isJavaFinalSigma(
+    const icu::UnicodeString& input,
+    int32_t sigmaOffset,
+    icu::BreakIterator& breakIterator) {
+  auto offset = sigmaOffset;
+  while (offset >= 0 && !isJavaWordBoundary(input, offset, breakIterator)) {
+    const auto codePoint = input.char32At(offset - 1);
+    if (isJavaCased(codePoint)) {
+      offset = sigmaOffset + 1;
+      while (offset < input.length() &&
+             !isJavaWordBoundary(input, offset, breakIterator)) {
+        const auto followingCodePoint = input.char32At(offset);
+        if (isJavaCased(followingCodePoint)) {
+          return false;
+        }
+        offset += U16_LENGTH(followingCodePoint);
+      }
+      return true;
+    }
+    offset -= U16_LENGTH(codePoint);
+  }
+  return false;
+}
+
+/// Holds one word break iterator per thread. BreakIterator::setText retains a
+/// reference to its input, hence the iterator is rebound to emptyText_ between
+/// calls before the caller's UnicodeString is modified or destroyed.
+class SparkBreakIteratorHolder {
+ public:
+  SparkBreakIteratorHolder() {
+    UErrorCode status = U_ZERO_ERROR;
+    const auto& locale = icu::Locale::getRoot();
+    breakIterator_.reset(
+        icu::BreakIterator::createWordInstance(locale, status));
+    BOLT_USER_CHECK(
+        U_SUCCESS(status) && breakIterator_ != nullptr,
+        "Failed to create BreakIterator: {}, for locale: {}, Data Path: {}",
+        u_errorName(status),
+        locale.getName(),
+        u_getDataDirectory());
+    breakIterator_->setText(emptyText_);
+  }
+
+  icu::BreakIterator& breakIterator() {
+    return *breakIterator_;
+  }
+
+  const icu::UnicodeString& emptyText() const {
+    return emptyText_;
+  }
+
+ private:
+  // Members are destroyed in reverse declaration order. Keep emptyText_ alive
+  // until after breakIterator_ releases its retained text reference.
+  icu::UnicodeString emptyText_;
+  std::unique_ptr<icu::BreakIterator> breakIterator_;
+};
+
+FOLLY_ALWAYS_INLINE SparkBreakIteratorHolder& sparkBreakIteratorHolder() {
+  static thread_local SparkBreakIteratorHolder holder;
+  return holder;
+}
+
+class ScopedBreakIteratorText {
+ public:
+  ScopedBreakIteratorText(
+      SparkBreakIteratorHolder& holder,
+      const icu::UnicodeString& input)
+      : holder_(holder) {
+    holder_.breakIterator().setText(input);
+  }
+
+  ~ScopedBreakIteratorText() {
+    holder_.breakIterator().setText(holder_.emptyText());
+  }
+
+  ScopedBreakIteratorText(const ScopedBreakIteratorText&) = delete;
+  ScopedBreakIteratorText& operator=(const ScopedBreakIteratorText&) = delete;
+
+ private:
+  SparkBreakIteratorHolder& holder_;
+};
+
+FOLLY_ALWAYS_INLINE void adjustJavaSigmaInPlace(
+    icu::UnicodeString& input,
+    int32_t sigmaOffset) {
+  struct Replacement {
+    int32_t offset;
+    char16_t codePoint;
+  };
+
+  folly::small_vector<Replacement, 4> replacements;
+  {
+    auto& holder = sparkBreakIteratorHolder();
+    ScopedBreakIteratorText scopedText(holder, input);
+    auto& breakIterator = holder.breakIterator();
+    do {
+      replacements.push_back(
+          {sigmaOffset,
+           isJavaFinalSigma(input, sigmaOffset, breakIterator)
+               ? char16_t{0x03C2}
+               : char16_t{0x03C3}});
+      sigmaOffset = input.indexOf(0x03A3, sigmaOffset + 1);
+    } while (sigmaOffset >= 0);
+  }
+
+  // BreakIterator no longer references input. It is now safe to mutate it.
+  for (const auto& replacement : replacements) {
+    input.setCharAt(replacement.offset, replacement.codePoint);
+  }
+}
+
+} // namespace bytedance::bolt::functions::stringCore::spark

bolt/functions/lib/string/StringCore.h

@@ -33,6 +33,7 @@
 #include <arm_sve.h>
 #endif
 #include <cstring>
+#include <memory>
 #include <string>
 #include <string_view>
 #include <utility>
@@ -52,6 +53,9 @@
 
 #include "bolt/common/base/Exceptions.h"
 #include "bolt/functions/lib/string/RegexUtils.h"
+#ifdef SPARK_COMPATIBLE
+#include "bolt/functions/lib/string/SparkLower.h"
+#endif
 #include "bolt/type/StringView.h"
 
 #if (ENABLE_VECTORIZATION > 0) && !defined(_DEBUG) && !defined(DEBUG)
@@ -1031,6 +1035,12 @@ FOLLY_ALWAYS_INLINE static std::string toLower(
   }
   icu::UnicodeString unicodeStr =
       icu::UnicodeString::fromUTF8({str, static_cast<int32_t>(length)});
+#ifdef SPARK_COMPATIBLE
+  const auto sigmaOffset = unicodeStr.indexOf(0x03A3);
+  if (sigmaOffset >= 0) {
+    spark::adjustJavaSigmaInPlace(unicodeStr, sigmaOffset);
+  }
+#endif
   unicodeStr.toLower(locale);
   result.clear();
   unicodeStr.toUTF8String(result);

bolt/functions/lib/string/tests/StringImplTest.cpp

@@ -1288,6 +1288,14 @@ TEST_F(StringImplTest, toLower) {
   EXPECT_EQ(
       toLower<true>(std::string_view("ΕΠΕΙΔΗ Η ΑΝΑΓΝΩΡΙΣΗ ΤΗΣ ΑΞΙΟΠΡΕΠΕΙΑΣ")),
       "επειδη η αναγνωριση της αξιοπρεπειας");
+#ifdef SPARK_COMPATIBLE
+  // Exercise repeated reuse of the thread-local BreakIterator, including more
+  // Sigma replacements than fit in the inline replacement buffer.
+  EXPECT_EQ(toLower<false>(std::string_view("AΣ8B")), "aσ8b");
+  EXPECT_EQ(
+      toLower<false>(std::string_view("AΣ AΣ AΣ AΣ AΣ")), "aς aς aς aς aς");
+  EXPECT_EQ(toLower<false>(std::string_view("AΣ")), "aς");
+#endif
   // Surrogate pairs.
   EXPECT_EQ(toLower<true>(std::string_view("a🙃b🙃c")), "a🙃b🙃c");
   EXPECT_EQ(toLower<true>(std::string_view("😀😆😃😄😄😆")), "😀😆😃😄😄😆");

bolt/functions/prestosql/StringFunctions.cpp

@@ -47,6 +47,7 @@ namespace bytedance::bolt::functions {
 using namespace stringCore;
 
 namespace {
+
 /**
  * Upper and Lower functions have a fast path for ascii where the functions
  * can be applied in place.
@@ -63,12 +64,17 @@ class UpperLowerTemplateFunction : public exec::VectorFunction {
         FlatVector<StringView>* results) {
       rows.applyToSelected([&](int row) {
         auto proxy = exec::StringWriter<>(results, row);
+        const auto input = decodedInput->valueAt<StringView>(row);
         if constexpr (isLower) {
-          stringImpl::lower<isAscii>(
-              proxy, decodedInput->valueAt<StringView>(row));
+          if constexpr (isAscii) {
+            stringImpl::lower<true>(proxy, input);
+          } else if (stringCore::isAscii(input.data(), input.size())) {
+            stringImpl::lower<true>(proxy, input);
+          } else {
+            stringImpl::lower<false>(proxy, input);
+          }
         } else {
-          stringImpl::upper<isAscii>(
-              proxy, decodedInput->valueAt<StringView>(row));
+          stringImpl::upper<isAscii>(proxy, input);
         }
         proxy.finalize();
       });

bolt/functions/sparksql/tests/StringTest.cpp

@@ -1582,6 +1582,76 @@ TEST_F(StringTest, lower) {
   // Ligatures.
   EXPECT_EQ(lower("ß fi ffi ff st ῗ"), "ß fi ffi ff st ῗ");
 }
+
+TEST_F(StringTest, lowerGreekSigmaCompatibility) {
+  struct TestCase {
+    int32_t id;
+    std::string input;
+    std::string defaultExpected;
+    std::string sparkExpected;
+  };
+
+  const std::vector<TestCase> testCases = {
+      {1, "Σ", "σ", "σ"},
+      {2, "ΣΣ", "σς", "σς"},
+      {3, "ΣΣ8", "σς8", "σς8"},
+      {4, "ΣΣ8I", "σς8i", "σσ8i"},
+      {5, "ΣΣI", "σσi", "σσi"},
+      {6, "AΣ", "aς", "aς"},
+      {7, "AΣ8", "aς8", "aς8"},
+      {8, "AΣ8B", "aς8b", "aσ8b"},
+      {9, "ΣA", "σa", "σa"},
+      {10, "8Σ", "8σ", "8σ"},
+      {11, "8Σ8", "8σ8", "8σ8"},
+      {12, "AΣ.", "aς.", "aς."},
+      {13, "AΣ-B", "aς-b", "aσ-b"},
+      {14, "AΣ_B", "aς_b", "aσ_b"},
+      {15,
+       "prefix ΣΣ46URZHBLJJ8",
+       "prefix σς46urzhbljj8",
+       "prefix σσ46urzhbljj8"},
+      {16,
+       "ΣΣ8IBLMR5OIM8 suffix",
+       "σς8iblmr5oim8 suffix",
+       "σσ8iblmr5oim8 suffix"},
+      {17, "ΣΑΛΑΤΑ", "σαλατα", "σαλατα"},
+      {18, "ΘΑΛΑΣΣΙΝΟΣ", "θαλασσινος", "θαλασσινος"},
+      {19, "ΟΣ", "ος", "ος"},
+      {20, "ΟΣ8", "ος8", "ος8"},
+      {21, "ΟΣΑ", "οσα", "οσα"},
+      {22, "σςΣ", "σςς", "σςς"},
+      {23, "Σ Σ", "σ σ", "σ σ"},
+      {24, "Σ-Σ", "σ-σ", "σ-ς"},
+      {25, "Σ.Σ", "σ.ς", "σ.ς"},
+      {26, "AΣ1B", "aς1b", "aσ1b"},
+      {27, "AΣ1", "aς1", "aς1"},
+      {28, "σς", "σς", "σς"},
+      {29, "AΣ@b", "aς@b", "aς@b"},
+      {30, "A@Σ", "a@σ", "a@σ"},
+      {31, "AΣ AΣ AΣ AΣ AΣ", "aς aς aς aς aς", "aς aς aς aς aς"},
+      {32, "AΣ²B", "aς²b", "aσ²b"},
+      {33, "AΣ·B", "aσ·b", "aς·b"},
+      {34, "AΣ가B", "aς가b", "aσ가b"},
+      {35, "AΣ㐀B", "aς㐀b", "aσ㐀b"},
+      {36, "AΣ\u200BB", "aσ\u200Bb", "aσ\u200Bb"},
+      {37, "AΣ\u202FB", "aς\u202Fb", "aς\u202Fb"},
+      {38, "A1-Σ", "a1-σ", "a1-σ"},
+      // Georgian Mkhedruli letters are cased in ICU 74, but not in Java 11.
+      {39, "AΣა", "aσა", "aςა"},
+      // Supplementary cased characters must be recognized without truncation.
+      {40, "AΣ𐐅", "aσ𐐭", "aσ𐐭"},
+  };
+
+  for (const auto& testCase : testCases) {
+    SCOPED_TRACE(testing::Message() << "id=" << testCase.id);
+#ifdef SPARK_COMPATIBLE
+    EXPECT_EQ(lower(testCase.input), testCase.sparkExpected);
+#else
+    EXPECT_EQ(lower(testCase.input), testCase.defaultExpected);
+#endif
+  }
+}
+
 TEST_F(StringTest, upper) {
   // Empty strings.
   EXPECT_EQ(upper(""), "");