Make whitespace filters Unicode-aware

Author: graygilmore

lib/liquid/standardfilters.rb

@@ -36,6 +36,16 @@ module StandardFilters
       %r{<style.*?</style>}m,
     )
     STRIP_HTML_TAGS = /<.*?>/m
+    UNICODE_WHITESPACE_LEFT = /\A[[:space:]]+/
+    UNICODE_WHITESPACE_RIGHT = /[[:space:]]+\z/
+    UNICODE_WHITESPACE_EDGES = Regexp.union(UNICODE_WHITESPACE_LEFT, UNICODE_WHITESPACE_RIGHT)
+    UNICODE_WHITESPACE_RUNS = /[[:space:]]+/
+    private_constant(
+      :UNICODE_WHITESPACE_EDGES,
+      :UNICODE_WHITESPACE_LEFT,
+      :UNICODE_WHITESPACE_RIGHT,
+      :UNICODE_WHITESPACE_RUNS,
+    )
 
     class << self
       def try_coerce_encoding(input, encoding:)
@@ -307,48 +317,56 @@ def split(input, pattern)
     # @liquid_category string
     # @liquid_summary
     #   Removes leading and trailing whitespace and collapses consecutive whitespace to a single space.
+    # @liquid_description
+    #   Whitespace is identified using the `[[:space:]]` regular expression character class.
     # @liquid_syntax string | squish
     # @liquid_return [string]
     def squish(input)
       return if input.nil?
 
-      Utils.to_s(input).strip.gsub(/\s+/, ' ')
+      Utils.to_s(input).gsub(UNICODE_WHITESPACE_RUNS, ' ').strip
     end
 
     # @liquid_public_docs
     # @liquid_type filter
     # @liquid_category string
     # @liquid_summary
     #   Strips all whitespace from the left and right of a string.
+    # @liquid_description
+    #   Whitespace is identified using the `[[:space:]]` regular expression character class.
     # @liquid_syntax string | strip
     # @liquid_return [string]
     def strip(input)
       input = Utils.to_s(input)
-      input.strip
+      input.gsub(UNICODE_WHITESPACE_EDGES, ' ').strip
     end
 
     # @liquid_public_docs
     # @liquid_type filter
     # @liquid_category string
     # @liquid_summary
     #   Strips all whitespace from the left of a string.
+    # @liquid_description
+    #   Whitespace is identified using the `[[:space:]]` regular expression character class.
     # @liquid_syntax string | lstrip
     # @liquid_return [string]
     def lstrip(input)
       input = Utils.to_s(input)
-      input.lstrip
+      input.gsub(UNICODE_WHITESPACE_LEFT, ' ').lstrip
     end
 
     # @liquid_public_docs
     # @liquid_type filter
     # @liquid_category string
     # @liquid_summary
     #   Strips all whitespace from the right of a string.
+    # @liquid_description
+    #   Whitespace is identified using the `[[:space:]]` regular expression character class.
     # @liquid_syntax string | rstrip
     # @liquid_return [string]
     def rstrip(input)
       input = Utils.to_s(input)
-      input.rstrip
+      input.gsub(UNICODE_WHITESPACE_RIGHT, ' ').rstrip
     end
 
     # @liquid_public_docs

test/integration/standard_filter_test.rb

@@ -169,6 +169,15 @@ def test_squish_filter
 \t   boo   " | squish }})).render)
     assert_equal("", Liquid::Template.parse('{{ nil | squish }}').render)
     assert_equal("", Liquid::Template.parse('{{ " " | squish }}').render)
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      "foo bar boo",
+      "{{ source | squish }}",
+      { 'source' => "#{unicode_spaces}foo\u202F\u2009bar\t\n\u2007boo#{unicode_spaces}" },
+    )
+    assert_template_result("\u200Bfoo\u200B", "{{ source | squish }}", { 'source' => "\u200Bfoo\u200B" })
   end
 
   def test_escape
@@ -703,16 +712,42 @@ def test_pipes_in_string_arguments
   def test_strip
     assert_template_result('ab c', "{{ source | strip }}", { 'source' => " ab c  " })
     assert_template_result('ab c', "{{ source | strip }}", { 'source' => " \tab c  \n \t" })
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      'ab c',
+      "{{ source | strip }}",
+      { 'source' => "#{unicode_spaces}ab c#{unicode_spaces}" },
+    )
+    assert_template_result("a\u00A0b\u202Fc", "{{ source | strip }}", { 'source' => "a\u00A0b\u202Fc" })
+    assert_template_result("\u200Bfoo\u200B", "{{ source | strip }}", { 'source' => "\u200Bfoo\u200B" })
   end
 
   def test_lstrip
     assert_template_result('ab c  ', "{{ source | lstrip }}", { 'source' => " ab c  " })
     assert_template_result("ab c  \n \t", "{{ source | lstrip }}", { 'source' => " \tab c  \n \t" })
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      "ab c#{unicode_spaces}",
+      "{{ source | lstrip }}",
+      { 'source' => "#{unicode_spaces}ab c#{unicode_spaces}" },
+    )
   end
 
   def test_rstrip
     assert_template_result(" ab c", "{{ source | rstrip }}", { 'source' => " ab c  " })
     assert_template_result(" \tab c", "{{ source | rstrip }}", { 'source' => " \tab c  \n \t" })
+
+    unicode_spaces = "\u00A0\u202F\u2009\u2007"
+
+    assert_template_result(
+      "#{unicode_spaces}ab c",
+      "{{ source | rstrip }}",
+      { 'source' => "#{unicode_spaces}ab c#{unicode_spaces}" },
+    )
   end
 
   def test_strip_newlines