diff --git a/sjsonnet/src/sjsonnet/Importer.scala b/sjsonnet/src/sjsonnet/Importer.scala index f5a2257d..ada0c391 100644 --- a/sjsonnet/src/sjsonnet/Importer.scala +++ b/sjsonnet/src/sjsonnet/Importer.scala @@ -380,9 +380,21 @@ object CachedResolver { case _: ujson.ParsingFailedException | _: DuplicateJsonKey | _: InvalidJsonNumber | _: JsonParseDepthExceeded | _: NumberFormatException => None + case e: Exception if isInvalidJsonSurrogateException(e) => + None } } + private def isInvalidJsonSurrogateException(e: Exception): Boolean = { + if (e.getClass != classOf[Exception]) return false + val message = e.getMessage + message != null && ( + message.startsWith("Unexpected character following high surrogate") || + message.startsWith("Duplicate high surrogate") || + message.startsWith("Un-paired low surrogate") + ) + } + private final class JsonImportVisitor( fileScope: FileScope, internedStrings: mutable.HashMap[String, String], diff --git a/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.json b/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.json new file mode 100644 index 00000000..53a3259f --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.json @@ -0,0 +1 @@ +{"key": "before\uDC00after"} diff --git a/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.jsonnet b/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.jsonnet new file mode 100644 index 00000000..b58a17c1 --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.jsonnet @@ -0,0 +1 @@ +import 'error.import_json_lone_low_surrogate.json' diff --git a/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.jsonnet.golden b/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.jsonnet.golden new file mode 100644 index 00000000..bafd4128 --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/error.import_json_lone_low_surrogate.jsonnet.golden @@ -0,0 +1,2 @@ +sjsonnet.ParseError: Expected "\"":1:16, found "\\uDC00afte" + at [].(error.import_json_lone_low_surrogate.jsonnet:1:1) diff --git a/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.json b/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.json new file mode 100644 index 00000000..d61ae2d1 --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.json @@ -0,0 +1 @@ +{"key": "\uDE00\uD83D"} diff --git a/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.jsonnet b/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.jsonnet new file mode 100644 index 00000000..221b7c5a --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.jsonnet @@ -0,0 +1 @@ +import 'error.import_json_reversed_surrogate.json' diff --git a/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.jsonnet.golden b/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.jsonnet.golden new file mode 100644 index 00000000..88f7069b --- /dev/null +++ b/sjsonnet/test/resources/new_test_suite/error.import_json_reversed_surrogate.jsonnet.golden @@ -0,0 +1,2 @@ +sjsonnet.ParseError: Expected "\"":1:10, found "\\uDE00\\uD8" + at [].(error.import_json_reversed_surrogate.jsonnet:1:1) diff --git a/sjsonnet/test/src/sjsonnet/JsonImportFastPathTests.scala b/sjsonnet/test/src/sjsonnet/JsonImportFastPathTests.scala index 16a85a89..0fa4bab0 100644 --- a/sjsonnet/test/src/sjsonnet/JsonImportFastPathTests.scala +++ b/sjsonnet/test/src/sjsonnet/JsonImportFastPathTests.scala @@ -97,6 +97,35 @@ object JsonImportFastPathTests extends TestSuite { } } + test("invalid unicode surrogate exceptions fall back without internal errors") { + val inputs = Map( + "high-before-non-low-escape.json" -> "{\"key\":\"\\uD800\\u0041\"}", + "duplicate-high.json" -> "{\"key\":\"\\uD800\\uD801\"}", + "lone-low.json" -> "{\"key\":\"before\\uDC00after\"}", + "reversed.json" -> "{\"key\":\"\\uDE00\\uD83D\"}" + ) + + inputs.foreach { case (fileName, json) => + val result = eval(Map(fileName -> json), s"""import "$fileName"""") + assert(result.isLeft) + result match { + case Left(error) => + assert(error.startsWith("sjsonnet.ParseError")) + assert(!error.contains("Internal Error")) + case Right(_) => assert(false) + } + } + } + + test("valid unicode surrogate pairs stay on the json fast path") { + val files = Map( + "unicode.json" -> "{\"emoji\":\"\\uD83D\\uDE00\"}" + ) + + eval(files, """import "unicode.json"""") ==> + Right(ujson.Obj("emoji" -> "\uD83D\uDE00")) + } + test("deep json imports keep parser recursion guard") { val files = Map("deep.json" -> """[[[]]]""")