diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/AllCodePointsTest.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/AllCodePointsTest.kt new file mode 100644 index 00000000000..b6a0488450f --- /dev/null +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/AllCodePointsTest.kt @@ -0,0 +1,53 @@ +import kotlin.text.* +import kotlin.test.* + +fun assertTrue(msg: String, value: Boolean) = assertTrue(value, msg) +fun assertFalse(msg: String, value: Boolean) = assertFalse(value, msg) + +fun codePointToString(codePoint: Int): String { + val charArray = Char.toChars(codePoint) + return fromCharArray(charArray, 0, charArray.size) +} + +fun box() {} + +// TODO: Here is a performance problem: an execution of this test requires much more time than it in Kotlin/JVM. +fun box1() { + // Regression for HARMONY-3145 + var p = Regex("(\\p{all})+") + var res = true + var cnt = 0 + var s: String + for (i in 0..1114111) { + if (i % 200000 == 0) { + println(i) + } + s = codePointToString(i) + // if (!s.matches(p.toString().toRegex())) { TODO: Uncomment when caching is done. + if (!s.matches(p)) { + cnt++ + res = false + } + } + assertTrue(res) + assertEquals(0, cnt) + + p = Regex("(\\P{all})+") + res = true + cnt = 0 + + for (i in 0..1114111) { + if (i % 200000 == 0) { + println(i) + } + s = codePointToString(i) + // if (!s.matches(p.toString().toRegex())) { TODO: Uncomment when caching is done. + if (!s.matches(p)) { + cnt++ + res = false + } + } + + assertFalse(res) + assertEquals(0x110000, cnt) +} \ No newline at end of file diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest.kt index ef335128f27..fffe31e64e6 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest.kt @@ -18,6 +18,9 @@ import kotlin.text.* import kotlin.test.* +fun assertTrue(msg: String, value: Boolean) = assertTrue(value, msg) +fun assertFalse(msg: String, value: Boolean) = assertFalse(value, msg) + internal var testPatterns = arrayOf("(a|b)*abb", "(1*2*3*4*)*567", "(a|b|c|d)*aab", "(1|2|3|4|5|6|7|8|9|0)(1|2|3|4|5|6|7|8|9|0)*", "(abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ)*", "(a|b)*(a|b)*A(a|b)*lice.*", "(a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z)(a|b|c|d|e|f|g|h|" + "i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z)*(1|2|3|4|5|6|7|8|9|0)*|while|for|struct|if|do") internal var groupPatterns = arrayOf("(a|b)*aabb", "((a)|b)*aabb", "((a|b)*)a(abb)", "(((a)|(b))*)aabb", "(((a)|(b))*)aa(b)b", "(((a)|(b))*)a(a(b)b)") @@ -78,7 +81,7 @@ fun testGroupint() { val regex = Regex(groupPatterns[i]) val result = regex.matchEntire(positiveTestString)!! for (j in 0..groupResults[i].size - 1) { - assertEquals("i: $i j: $j", groupResults[i][j], result.groupValues[j + 1]) + assertEquals(groupResults[i][j], result.groupValues[j + 1], "i: $i j: $j") } } @@ -411,42 +414,6 @@ fun testFindDollar() { assertEquals("a", result!!.groupValues[0]) } - -fun testAllCodePoints() { - // Regression for HARMONY-3145 - val codePoint = IntArray(1) - var p = Regex("(\\p{all})+") - var res = true - var cnt = 0 - var s: String - for (i in 0..1114111) { - codePoint[0] = i - s = String(codePoint, 0, 1) - if (!s.matches(p.toString().toRegex())) { - cnt++ - res = false - } - } - assertTrue(res) - assertEquals(0, cnt) - - p = Regex("(\\P{all})+") - res = true - cnt = 0 - - for (i in 0..1114111) { - codePoint[0] = i - s = String(codePoint, 0, 1) - if (!s.matches(p.toString().toRegex())) { - cnt++ - res = false - } - } - - assertFalse(res) - assertEquals(0x110000, cnt) -} - /* * Regression test for HARMONY-674 */ @@ -501,7 +468,6 @@ fun box() { testUnicodeCategory() testSplitEmpty() testFindDollar() - testAllCodePoints() testPatternMatcher() test3360() testGeneralPunctuationCategory() diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest2.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest2.kt index 0c53b19436b..d39f25e2a7d 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest2.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/MatchResultTest2.kt @@ -22,77 +22,77 @@ fun testErrorConditions2() { val regex = Regex("(foo[0-9])(bar[a-z])") var result = regex.find("foo1barzfoo2baryfoozbar5") - Assert.assertTrue(result != null) - Assert.assertEquals(0, result!!.range.start) - Assert.assertEquals(7, result.range.endInclusive) - Assert.assertEquals(0, result.groups[0]!!.range.start) - Assert.assertEquals(7, result.groups[0]!!.range.endInclusive) - Assert.assertEquals(0, result.groups[1]!!.range.start) - Assert.assertEquals(3, result.groups[1]!!.range.endInclusive) - Assert.assertEquals(4, result.groups[2]!!.range.start) - Assert.assertEquals(7, result.groups[2]!!.range.endInclusive) + assertTrue(result != null) + assertEquals(0, result!!.range.start) + assertEquals(7, result.range.endInclusive) + assertEquals(0, result.groups[0]!!.range.start) + assertEquals(7, result.groups[0]!!.range.endInclusive) + assertEquals(0, result.groups[1]!!.range.start) + assertEquals(3, result.groups[1]!!.range.endInclusive) + assertEquals(4, result.groups[2]!!.range.start) + assertEquals(7, result.groups[2]!!.range.endInclusive) try { result.groups[3] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } try { result.groupValues[3] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } try { result.groups[-1] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } try { result.groupValues[-1] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } result = result.next() - Assert.assertTrue(result != null) - Assert.assertEquals(8, result!!.range.start) - Assert.assertEquals(15, result.range.endInclusive) - Assert.assertEquals(8, result.groups[0]!!.range.start) - Assert.assertEquals(15, result.groups[0]!!.range.endInclusive) - Assert.assertEquals(8, result.groups[1]!!.range.start) - Assert.assertEquals(11, result.groups[1]!!.range.endInclusive) - Assert.assertEquals(12, result.groups[2]!!.range.start) - Assert.assertEquals(15, result.groups[2]!!.range.endInclusive) + assertTrue(result != null) + assertEquals(8, result!!.range.start) + assertEquals(15, result.range.endInclusive) + assertEquals(8, result.groups[0]!!.range.start) + assertEquals(15, result.groups[0]!!.range.endInclusive) + assertEquals(8, result.groups[1]!!.range.start) + assertEquals(11, result.groups[1]!!.range.endInclusive) + assertEquals(12, result.groups[2]!!.range.start) + assertEquals(15, result.groups[2]!!.range.endInclusive) try { result.groups[3] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } try { result.groupValues[3] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } try { result.groups[-1] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } try { result.groupValues[-1] - Assert.fail("IndexOutOfBoundsException expected") + fail("IndexOutOfBoundsException expected") } catch (e: IndexOutOfBoundsException) { } result = result.next() - Assert.assertFalse(result != null) + assertFalse(result != null) } /* @@ -105,7 +105,7 @@ fun testReplacementBackSlash() { val regex = Regex(replacedString) try { regex.replace(str, substitutionString) - Assert.fail("IllegalArgumentException should be thrown") + fail("IllegalArgumentException should be thrown") } catch (e: IllegalArgumentException) { } } diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/ModeTest.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/ModeTest.kt index e36b1c6528f..8e7f46db163 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/ModeTest.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/ModeTest.kt @@ -26,27 +26,27 @@ fun testCase() { regex = Regex("([a-z]+)[0-9]+") result = regex.find("cAT123#dog345") - Assert.assertNotNull(result) - Assert.assertEquals("dog", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("dog", result!!.groupValues[1]) + assertNull(result.next()) regex = Regex("([a-z]+)[0-9]+", RegexOption.IGNORE_CASE) result = regex.find("cAt123#doG345") - Assert.assertNotNull(result) - Assert.assertEquals("cAt", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("cAt", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("doG", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("doG", result!!.groupValues[1]) + assertNull(result.next()) regex = Regex("(?i)([a-z]+)[0-9]+") result = regex.find("cAt123#doG345") - Assert.assertNotNull(result) - Assert.assertEquals("cAt", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("cAt", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("doG", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("doG", result!!.groupValues[1]) + assertNull(result.next()) } fun testMultiline() { @@ -55,59 +55,59 @@ fun testMultiline() { regex = Regex("^foo") result = regex.find("foobar") - Assert.assertNotNull(result) - Assert.assertTrue(result!!.range.start == 0 && result.range.endInclusive == 2) - Assert.assertTrue(result.groups[0]!!.range.start == 0 && result.groups[0]!!.range.endInclusive == 2) - Assert.assertNull(result.next()) + assertNotNull(result) + assertTrue(result!!.range.start == 0 && result.range.endInclusive == 2) + assertTrue(result.groups[0]!!.range.start == 0 && result.groups[0]!!.range.endInclusive == 2) + assertNull(result.next()) result = regex.find("barfoo") - Assert.assertNull(result) + assertNull(result) regex = Regex("foo$") result = regex.find("foobar") - Assert.assertNull(result) + assertNull(result) result = regex.find("barfoo") - Assert.assertNotNull(result) - Assert.assertTrue(result!!.range.start == 3 && result.range.endInclusive == 5) - Assert.assertTrue(result.groups[0]!!.range.start == 3 && result.groups[0]!!.range.endInclusive == 5) - Assert.assertNull(result.next()) + assertNotNull(result) + assertTrue(result!!.range.start == 3 && result.range.endInclusive == 5) + assertTrue(result.groups[0]!!.range.start == 3 && result.groups[0]!!.range.endInclusive == 5) + assertNull(result.next()) regex = Regex("^foo([0-9]*)", RegexOption.MULTILINE) result = regex.find("foo1bar\nfoo2foo3\nbarfoo4") - Assert.assertNotNull(result) - Assert.assertEquals("1", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("1", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("2", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("2", result!!.groupValues[1]) + assertNull(result.next()) regex = Regex("foo([0-9]*)$", RegexOption.MULTILINE) result = regex.find("foo1bar\nfoo2foo3\nbarfoo4") - Assert.assertNotNull(result) - Assert.assertEquals("3", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("3", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("4", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("4", result!!.groupValues[1]) + assertNull(result.next()) regex = Regex("(?m)^foo([0-9]*)") result = regex.find("foo1bar\nfoo2foo3\nbarfoo4") - Assert.assertNotNull(result) - Assert.assertEquals("1", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("1", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("2", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("2", result!!.groupValues[1]) + assertNull(result.next()) regex = Regex("(?m)foo([0-9]*)$") result = regex.find("foo1bar\nfoo2foo3\nbarfoo4") - Assert.assertNotNull(result) - Assert.assertEquals("3", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("3", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("4", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("4", result!!.groupValues[1]) + assertNull(result.next()) } fun box() { diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternSyntaxExceptionTest.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternSyntaxExceptionTest.kt index 3ac44315eb5..346c6514b20 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternSyntaxExceptionTest.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternSyntaxExceptionTest.kt @@ -21,7 +21,7 @@ fun testCase() { val regex = "(" try { Regex(regex) - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { // TODO: Check the exception's properties. } @@ -32,7 +32,7 @@ fun testCase2() { val regex = "[4-" try { Regex(regex) - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest.kt index 920bcdbd7b8..8622887011b 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest.kt @@ -18,71 +18,74 @@ import kotlin.text.* import kotlin.test.* +fun assertTrue(msg: String, value: Boolean) = assertTrue(value, msg) +fun assertFalse(msg: String, value: Boolean) = assertFalse(value, msg) + internal var testPatterns = arrayOf("(a|b)*abb", "(1*2*3*4*)*567", "(a|b|c|d)*aab", "(1|2|3|4|5|6|7|8|9|0)(1|2|3|4|5|6|7|8|9|0)*", "(abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ)*", "(a|b)*(a|b)*A(a|b)*lice.*", "(a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z)(a|b|c|d|e|f|g|h|" + "i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z)*(1|2|3|4|5|6|7|8|9|0)*|while|for|struct|if|do", "x(?c)y", "x(?cc)y", "x(?:c)y") fun testCommentsInPattern() { val p = Regex("ab# this is a comment\ncd", RegexOption.COMMENTS) - Assert.assertTrue(p.matches("abcd")) + assertTrue(p.matches("abcd")) } fun testSplitCharSequenceint() { // Splitting CharSequence which ends with pattern. // Harmony regress tests. - Assert.assertEquals(",,".split(",".toRegex(), 3).toTypedArray().size, 3) - Assert.assertEquals(",,".split(",".toRegex(), 4).toTypedArray().size, 3) - Assert.assertEquals(Regex("o").split("boo:and:foo", 5).size, 5) - Assert.assertEquals(Regex("b").split("ab", 0).size, 2) + assertEquals(",,".split(",".toRegex(), 3).toTypedArray().size, 3) + assertEquals(",,".split(",".toRegex(), 4).toTypedArray().size, 3) + assertEquals(Regex("o").split("boo:and:foo", 5).size, 5) + assertEquals(Regex("b").split("ab", 0).size, 2) var s: List var regex = Regex("x") s = regex.split("zxx:zzz:zxx", 10) - Assert.assertEquals(s.size, 5) + assertEquals(s.size, 5) s = regex.split("zxx:zzz:zxx", 3) - Assert.assertEquals(s.size, 3) + assertEquals(s.size, 3) s = regex.split("zxx:zzz:zxx", 0) - Assert.assertEquals(s.size, 5) + assertEquals(s.size, 5) // Other splitting. // Negative limit regex = Regex("b") s = regex.split("abccbadfebb", 0) - Assert.assertEquals(s.size, 5) + assertEquals(s.size, 5) s = regex.split("", 0) - Assert.assertEquals(s.size, 1) + assertEquals(s.size, 1) regex = Regex("") s = regex.split("", 0) - Assert.assertEquals(s.size, 1) + assertEquals(s.size, 1) s = regex.split("abccbadfe", 0) - Assert.assertEquals(s.size, 11) + assertEquals(s.size, 11) // positive limit regex = Regex("b") s = regex.split("abccbadfebb", 12) - Assert.assertEquals(s.size, 5) + assertEquals(s.size, 5) s = regex.split("", 6) - Assert.assertEquals(s.size, 1) + assertEquals(s.size, 1) regex = Regex("") s = regex.split("", 11) - Assert.assertEquals(s.size, 1) + assertEquals(s.size, 1) s = regex.split("abccbadfe", 15) - Assert.assertEquals(s.size, 11) + assertEquals(s.size, 11) regex = Regex("b") s = regex.split("abccbadfebb", 5) - Assert.assertEquals(s.size, 5) + assertEquals(s.size, 5) s = regex.split("", 1) - Assert.assertEquals(s.size, 1) + assertEquals(s.size, 1) regex = Regex("") s = regex.split("", 1) - Assert.assertEquals(s.size, 1) + assertEquals(s.size, 1) s = regex.split("abccbadfe", 11) - Assert.assertEquals(s.size, 11) + assertEquals(s.size, 11) regex = Regex("b") s = regex.split("abccbadfebb", 3) - Assert.assertEquals(s.size, 3) + assertEquals(s.size, 3) regex = Regex("") s = regex.split("abccbadfe", 5) - Assert.assertEquals(s.size, 5) + assertEquals(s.size, 5) } fun testFlags() { @@ -93,98 +96,98 @@ fun testFlags() { baseString = "((?i)|b)a" testString = "A" regex = Regex(baseString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) baseString = "(?i)a|b" testString = "A" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)a|b" testString = "B" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "c|(?i)a|b" testString = "B" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)a|(?s)b" testString = "B" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)a|(?-i)b" testString = "B" regex = Regex(baseString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) baseString = "(?i)a|(?-i)c|b" testString = "B" regex = Regex(baseString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) baseString = "(?i)a|(?-i)c|(?i)b" testString = "B" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)a|(?-i)b" testString = "A" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "((?i))a" testString = "A" regex = Regex(baseString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) baseString = "|(?i)|a" testString = "A" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)((?s)a.)" testString = "A\n" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)((?-i)a)" testString = "A" regex = Regex(baseString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) baseString = "(?i)(?s:a.)" testString = "A\n" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)fgh(?s:aa)" testString = "fghAA" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?i)((?-i))a" testString = "A" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "abc(?i)d" testString = "ABCD" regex = Regex(baseString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) testString = "abcD" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "a(?i)a(?-i)a(?i)a(?-i)a" testString = "aAaAa" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "aAAAa" - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) } fun Set.containsOnly(vararg options: RegexOption): Boolean { @@ -202,26 +205,26 @@ fun testFlagsMethod() { baseString = "(?idmsux)abc(?-i)vg(?-dmu)" regex = Regex(baseString) - Assert.assertTrue(regex.options.containsOnly(RegexOption.DOT_MATCHES_ALL, RegexOption.COMMENTS)) + assertTrue(regex.options.containsOnly(RegexOption.DOT_MATCHES_ALL, RegexOption.COMMENTS)) baseString = "(?idmsux)abc|(?-i)vg|(?-dmu)" regex = Regex(baseString) - Assert.assertTrue(regex.options.containsOnly(RegexOption.DOT_MATCHES_ALL, RegexOption.COMMENTS)) + assertTrue(regex.options.containsOnly(RegexOption.DOT_MATCHES_ALL, RegexOption.COMMENTS)) baseString = "(?is)a((?x)b.)" regex = Regex(baseString) - Assert.assertTrue(regex.options.containsOnly(RegexOption.DOT_MATCHES_ALL, RegexOption.IGNORE_CASE)) + assertTrue(regex.options.containsOnly(RegexOption.DOT_MATCHES_ALL, RegexOption.IGNORE_CASE)) baseString = "(?i)a((?-i))" regex = Regex(baseString) - Assert.assertTrue(regex.options.containsOnly(RegexOption.IGNORE_CASE)) + assertTrue(regex.options.containsOnly(RegexOption.IGNORE_CASE)) baseString = "((?i)a)" regex = Regex(baseString) - Assert.assertTrue(regex.options.isEmpty()) + assertTrue(regex.options.isEmpty()) regex = Regex("(?is)abc") - Assert.assertTrue(regex.options.containsOnly(RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL)) + assertTrue(regex.options.containsOnly(RegexOption.IGNORE_CASE, RegexOption.DOT_MATCHES_ALL)) } fun testCompileStringint() { @@ -231,7 +234,7 @@ fun testCompileStringint() { var pattern = "b)a" try { Regex(pattern) - Assert.fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) + fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) } catch (e: PatternSyntaxException) { // pass } @@ -239,7 +242,7 @@ fun testCompileStringint() { pattern = "bcde)a" try { Regex(pattern) - Assert.fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) + fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) } catch (e: PatternSyntaxException) { // pass } @@ -247,7 +250,7 @@ fun testCompileStringint() { pattern = "bbg())a" try { Regex(pattern) - Assert.fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) + fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) } catch (e: PatternSyntaxException) { // pass } @@ -255,7 +258,7 @@ fun testCompileStringint() { pattern = "cdb(?i))a" try { Regex(pattern) - Assert.fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) + fail("Expected a PatternSyntaxException when compiling pattern: " + pattern) } catch (e: PatternSyntaxException) { // pass } @@ -279,7 +282,7 @@ fun testQuantCompileNeg() { for (element in patterns) { try { Regex(element) - Assert.fail("PatternSyntaxException was expected, but compilation succeeds") + fail("PatternSyntaxException was expected, but compilation succeeds") } catch (pse: PatternSyntaxException) { continue } @@ -298,19 +301,19 @@ fun testQuantComposition() { val pattern = "(a{1,3})aab" val regex = Regex(pattern) val result = regex.matchEntire("aaab") - Assert.assertNotNull(result) - Assert.assertEquals(result!!.groups[1]!!.range.start, 0) - Assert.assertEquals(result.groupValues[1], "a") + assertNotNull(result) + assertEquals(result!!.groups[1]!!.range.start, 0) + assertEquals(result.groupValues[1], "a") } fun testTimeZoneIssue() { val regex = Regex("GMT(\\+|\\-)(\\d+)(:(\\d+))?") val result = regex.matchEntire("GMT-9:45") - Assert.assertNotNull(result) - Assert.assertEquals("-", result!!.groupValues[1]) - Assert.assertEquals("9", result.groupValues[2]) - Assert.assertEquals(":45", result.groupValues[3]) - Assert.assertEquals("45", result.groupValues[4]) + assertNotNull(result) + assertEquals("-", result!!.groupValues[1]) + assertEquals("9", result.groupValues[2]) + assertEquals(":45", result.groupValues[3]) + assertEquals("45", result.groupValues[4]) } fun testCompileRanges() { @@ -323,7 +326,7 @@ fun testCompileRanges() { "jhfkjhaSDFGHJkdfhHNJMjkhfabb", "+*??+*abb", "sdfghjkabb") for (i in correctTestPatterns.indices) { - Assert.assertTrue("pattern: " + correctTestPatterns[i] + " input: " + inputSecuence[i], + assertTrue("pattern: " + correctTestPatterns[i] + " input: " + inputSecuence[i], Regex(correctTestPatterns[i]).matches(inputSecuence[i])) } @@ -332,7 +335,7 @@ fun testCompileRanges() { "jhfkjhaSDFGHJk;dfhHNJMjkhfabb", "+*?a?+*abb", "sdf+ghjkabb") for (i in correctTestPatterns.indices) { - Assert.assertFalse("pattern: " + correctTestPatterns[i] + " input: " + wrongInputSecuence[i], + assertFalse("pattern: " + correctTestPatterns[i] + " input: " + wrongInputSecuence[i], Regex(correctTestPatterns[i]).matches(wrongInputSecuence[i])) } } @@ -343,7 +346,7 @@ fun testRangesSpecialCases() { for (element in neg_patterns) { try { Regex(element) - Assert.fail("PatternSyntaxException was expected: " + element) + fail("PatternSyntaxException was expected: " + element) } catch (pse: PatternSyntaxException) { } @@ -355,18 +358,18 @@ fun testRangesSpecialCases() { while (i < pos_patterns.size) { val pat = pos_patterns[i++] val inp = pos_patterns[i] - Assert.assertTrue("pattern: $pat input: $inp", Regex(pat).matches(inp)) + assertTrue("pattern: $pat input: $inp", Regex(pat).matches(inp)) i++ } } fun testZeroSymbols() { - Assert.assertTrue(Regex("[\u0000]*abb").matches("\u0000\u0000\u0000\u0000\u0000\u0000abb")) + assertTrue(Regex("[\u0000]*abb").matches("\u0000\u0000\u0000\u0000\u0000\u0000abb")) } fun testEscapes() { val regex = Regex("\\Q{]()*?") - Assert.assertTrue(regex.matches("{]()*?")) + assertTrue(regex.matches("{]()*?")) } fun testRegressions() { @@ -386,7 +389,7 @@ fun testRegressions() { fun testOrphanQuantifiers() { try { Regex("+++++") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (pse: PatternSyntaxException) { } @@ -395,7 +398,7 @@ fun testOrphanQuantifiers() { fun testOrphanQuantifiers2() { try { Regex("\\d+*") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (pse: PatternSyntaxException) { } @@ -417,10 +420,10 @@ fun testBug197() { val res = Regex(vals[i++].toString()).split("boo:and:foo", (vals[i++] as Int)) val expectedRes = vals[i++] as Array - Assert.assertEquals(expectedRes.size, res.size) + assertEquals(expectedRes.size, res.size) for (j in expectedRes.indices) { - Assert.assertEquals(expectedRes[j], res[j]) + assertEquals(expectedRes[j], res[j]) } } } @@ -447,16 +450,16 @@ fun testFindBoundaryCases1() { val regex = Regex(".*\n") val result = regex.find("a\n") - Assert.assertNotNull(result) - Assert.assertEquals("a\n", result!!.value) + assertNotNull(result) + assertEquals("a\n", result!!.value) } fun testFindBoundaryCases2() { val regex = Regex(".*A") val result = regex.find("aAa") - Assert.assertNotNull(result) - Assert.assertEquals("aA", result!!.value) + assertNotNull(result) + assertEquals("aA", result!!.value) } @@ -464,8 +467,8 @@ fun testFindBoundaryCases3() { val regex = Regex(".*A") val result = regex.find("a\naA\n") - Assert.assertNotNull(result) - Assert.assertEquals("aA", result!!.value) + assertNotNull(result) + assertEquals("aA", result!!.value) } @@ -473,8 +476,8 @@ fun testFindBoundaryCases4() { val regex = Regex("A.*") val result = regex.find("A\n") - Assert.assertNotNull(result) - Assert.assertEquals("A", result!!.value) + assertNotNull(result) + assertEquals("A", result!!.value) } @@ -485,7 +488,7 @@ fun testFindBoundaryCases5() { var k = 0 while (result != null) { - Assert.assertEquals(expected[k], result.value) + assertEquals(expected[k], result.value) result = result.next() k++ } @@ -498,7 +501,7 @@ fun testFindBoundaryCases6() { var k = 0 while (result != null) { - Assert.assertEquals(expected[k], result.value) + assertEquals(expected[k], result.value) k++ result = result.next() } @@ -510,16 +513,16 @@ fun testBackReferences() { var k = 1 while (result != null) { - Assert.assertEquals("start" + k, result.groupValues[2]) - Assert.assertEquals(" word ", result.groupValues[3]) - Assert.assertEquals("start" + k, result.groupValues[4]) + assertEquals("start" + k, result.groupValues[2]) + assertEquals(" word ", result.groupValues[3]) + assertEquals("start" + k, result.groupValues[4]) k++ result = result.next() } - Assert.assertEquals(3, k) + assertEquals(3, k) regex = Regex(".*(.)\\1") - Assert.assertTrue(regex.matches("saa")) + assertTrue(regex.matches("saa")) } fun testNewLine() { @@ -530,14 +533,14 @@ fun testNewLine() { counter++ result = result.next() } - Assert.assertEquals(2, counter) + assertEquals(2, counter) } fun testFindGreedy() { val regex = Regex(".*aaa", RegexOption.DOT_MATCHES_ALL) val result = regex.matchEntire("aaaa\naaa\naaaaaa") - Assert.assertNotNull(result) - Assert.assertEquals(14, result!!.range.endInclusive) + assertNotNull(result) + assertEquals(14, result!!.range.endInclusive) } fun testSOLQuant() { @@ -548,13 +551,13 @@ fun testSOLQuant() { counter++ result = result.next() } - Assert.assertEquals(3, counter) + assertEquals(3, counter) } fun testIllegalEscape() { try { Regex("\\y") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (pse: PatternSyntaxException) { } } @@ -566,164 +569,164 @@ fun testEmptyFamily() { fun testNonCaptConstr() { // Flags var regex = Regex("(?i)b*(?-i)a*") - Assert.assertTrue(regex.matches("bBbBaaaa")) - Assert.assertFalse(regex.matches("bBbBAaAa")) + assertTrue(regex.matches("bBbBaaaa")) + assertFalse(regex.matches("bBbBAaAa")) // Non-capturing groups regex = Regex("(?i:b*)a*") - Assert.assertTrue(regex.matches("bBbBaaaa")) - Assert.assertFalse(regex.matches("bBbBAaAa")) + assertTrue(regex.matches("bBbBaaaa")) + assertFalse(regex.matches("bBbBAaAa")) // 1 2 3 4 5 6 7 8 9 10 11 regex = Regex("(?:-|(-?\\d+\\d\\d\\d))?(?:-|-(\\d\\d))?(?:-|-(\\d\\d))?(T)?(?:(\\d\\d):(\\d\\d):(\\d\\d)(\\.\\d+)?)?(?:(?:((?:\\+|\\-)\\d\\d):(\\d\\d))|(Z))?") val result = regex.matchEntire("-1234-21-31T41:51:61.789+71:81") - Assert.assertNotNull(result) - Assert.assertEquals("-1234", result!!.groupValues[1]) - Assert.assertEquals("21", result.groupValues[2]) - Assert.assertEquals("31", result.groupValues[3]) - Assert.assertEquals("T", result.groupValues[4]) - Assert.assertEquals("41", result.groupValues[5]) - Assert.assertEquals("51", result.groupValues[6]) - Assert.assertEquals("61", result.groupValues[7]) - Assert.assertEquals(".789", result.groupValues[8]) - Assert.assertEquals("+71", result.groupValues[9]) - Assert.assertEquals("81", result.groupValues[10]) + assertNotNull(result) + assertEquals("-1234", result!!.groupValues[1]) + assertEquals("21", result.groupValues[2]) + assertEquals("31", result.groupValues[3]) + assertEquals("T", result.groupValues[4]) + assertEquals("41", result.groupValues[5]) + assertEquals("51", result.groupValues[6]) + assertEquals("61", result.groupValues[7]) + assertEquals(".789", result.groupValues[8]) + assertEquals("+71", result.groupValues[9]) + assertEquals("81", result.groupValues[10]) // positive lookahead regex = Regex(".*\\.(?=log$).*$") - Assert.assertTrue(regex.matches("a.b.c.log")) - Assert.assertFalse(regex.matches("a.b.c.log.")) + assertTrue(regex.matches("a.b.c.log")) + assertFalse(regex.matches("a.b.c.log.")) // negative lookahead regex = Regex(".*\\.(?!log$).*$") - Assert.assertFalse(regex.matches("abc.log")) - Assert.assertTrue(regex.matches("abc.logg")) + assertFalse(regex.matches("abc.log")) + assertTrue(regex.matches("abc.logg")) // positive lookbehind regex = Regex(".*(?<=abc)\\.log$") - Assert.assertFalse(regex.matches("cde.log")) - Assert.assertTrue(regex.matches("abc.log")) + assertFalse(regex.matches("cde.log")) + assertTrue(regex.matches("abc.log")) // negative lookbehind regex = Regex(".*(?a*)abb") - Assert.assertFalse(regex.matches("aaabb")) + assertFalse(regex.matches("aaabb")) regex = Regex("(?>a*)bb") - Assert.assertTrue(regex.matches("aaabb")) + assertTrue(regex.matches("aaabb")) regex = Regex("(?>a|aa)aabb") - Assert.assertTrue(regex.matches("aaabb")) + assertTrue(regex.matches("aaabb")) regex = Regex("(?>aa|a)aabb") - Assert.assertFalse(regex.matches("aaabb")) + assertFalse(regex.matches("aaabb")) // quantifiers over look ahead regex = Regex(".*(?<=abc)*\\.log$") - Assert.assertTrue(regex.matches("cde.log")) + assertTrue(regex.matches("cde.log")) regex = Regex(".*(?<=abc)+\\.log$") - Assert.assertFalse(regex.matches("cde.log")) + assertFalse(regex.matches("cde.log")) } fun testCompilePatternWithTerminatorMark() { val regex = Regex("a\u0000\u0000cd") - Assert.assertTrue(regex.matches("a\u0000\u0000cd")) + assertTrue(regex.matches("a\u0000\u0000cd")) } fun testAlternations() { var baseString = "|a|bc" var regex = Regex(baseString) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) baseString = "a||bc" regex = Regex(baseString) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) baseString = "a|bc|" regex = Regex(baseString) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) baseString = "a|b|" regex = Regex(baseString) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) baseString = "a(|b|cd)e" regex = Regex(baseString) - Assert.assertTrue(regex.matches("ae")) + assertTrue(regex.matches("ae")) baseString = "a(b||cd)e" regex = Regex(baseString) - Assert.assertTrue(regex.matches("ae")) + assertTrue(regex.matches("ae")) baseString = "a(b|cd|)e" regex = Regex(baseString) - Assert.assertTrue(regex.matches("ae")) + assertTrue(regex.matches("ae")) baseString = "a(b|c|)e" regex = Regex(baseString) - Assert.assertTrue(regex.matches("ae")) + assertTrue(regex.matches("ae")) baseString = "a(|)e" regex = Regex(baseString) - Assert.assertTrue(regex.matches("ae")) + assertTrue(regex.matches("ae")) baseString = "|" regex = Regex(baseString) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) baseString = "a(?:|)e" regex = Regex(baseString) - Assert.assertTrue(regex.matches("ae")) + assertTrue(regex.matches("ae")) baseString = "a||||bc" regex = Regex(baseString) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) baseString = "(?i-is)|a" regex = Regex(baseString) - Assert.assertTrue(regex.matches("a")) + assertTrue(regex.matches("a")) } fun testMatchWithGroups() { var baseString = "jwkerhjwehrkwjehrkwjhrwkjehrjwkehrjkwhrkwehrkwhrkwrhwkhrwkjehr" var pattern = ".*(..).*\\1.*" - Assert.assertTrue(Regex(pattern).matches(baseString)) + assertTrue(Regex(pattern).matches(baseString)) baseString = "saa" pattern = ".*(.)\\1" - Assert.assertTrue(Regex(pattern).matches(baseString)) - Assert.assertTrue(Regex(pattern).containsMatchIn(baseString)) + assertTrue(Regex(pattern).matches(baseString)) + assertTrue(Regex(pattern).containsMatchIn(baseString)) } fun testSplitEmptyCharSequence() { val s1 = "" val arr = s1.split(":".toRegex()) - Assert.assertEquals(arr.size, 1) + assertEquals(arr.size, 1) } fun testSplitEndsWithPattern() { - Assert.assertEquals(",,".split(",".toRegex(), 3).toTypedArray().size, 3) - Assert.assertEquals(",,".split(",".toRegex(), 4).toTypedArray().size, 3) + assertEquals(",,".split(",".toRegex(), 3).toTypedArray().size, 3) + assertEquals(",,".split(",".toRegex(), 4).toTypedArray().size, 3) - Assert.assertEquals(Regex("o").split("boo:and:foo", 5).size, 5) - Assert.assertEquals(Regex("b").split("ab", 0).size, 2) + assertEquals(Regex("o").split("boo:and:foo", 5).size, 5) + assertEquals(Regex("b").split("ab", 0).size, 2) } fun testCaseInsensitiveFlag() { - Assert.assertTrue(Regex("(?i-:AbC)").matches("ABC")) + assertTrue(Regex("(?i-:AbC)").matches("ABC")) } fun testEmptyGroups() { var regex = Regex("ab(?>)cda") - Assert.assertTrue(regex.matches("abcda")) + assertTrue(regex.matches("abcda")) regex = Regex("ab()") - Assert.assertTrue(regex.matches("ab")) + assertTrue(regex.matches("ab")) regex = Regex("abc(?:)(..)") - Assert.assertTrue(regex.matches("abcgf")) + assertTrue(regex.matches("abcgf")) } fun testCompileNonCaptGroup() { @@ -738,34 +741,34 @@ fun testCompileNonCaptGroup() { } catch (e: PatternSyntaxException) { println(e) } - Assert.assertTrue(isCompiled) + assertTrue(isCompiled) } fun testEmbeddedFlags() { var baseString = "(?i)((?s)a)" var testString = "A" var regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?x)(?i)(?s)(?d)a" testString = "A" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "(?x)(?i)(?s)(?d)a." testString = "a\n" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "abc(?x:(?i)(?s)(?d)a.)" testString = "abcA\n" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "abc((?x)d)(?i)(?s)a" testString = "abcdA" regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) } fun testAltWithFlags() { @@ -776,7 +779,7 @@ fun testRestoreFlagsAfterGroup() { val baseString = "abc((?x)d) a" val testString = "abcd a" val regex = Regex(baseString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) } fun testCanonEqFlag() { @@ -805,50 +808,50 @@ fun testCanonEqFlag() { baseString = "\u01E0\u00CCcdb(ac)" testString = "\u0226\u0304\u0049\u0300cdbac" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\u01E0cdb(a\u00CCc)" testString = "\u0041\u0307\u0304cdba\u0049\u0300c" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "a\u00CC" testString = "a\u0049\u0300" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\u0226\u0304cdb(ac\u0049\u0300)" testString = "\u01E0cdbac\u00CC" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "cdb(?:\u0041\u0307\u0304\u00CC)" testString = "cdb\u0226\u0304\u0049\u0300" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\u01E0[a-c]\u0049\u0300cdb(ac)" testString = "\u01E0b\u00CCcdbac" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\u01E0|\u00CCcdb(ac)" testString = "\u0041\u0307\u0304" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\u00CC?cdb(ac)*(\u01E0)*[a-c]" testString = "cdb\u0041\u0307\u0304b" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "a\u0300" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.containsMatchIn("a\u00E0a")) + assertTrue(regex.containsMatchIn("a\u00E0a")) baseString = "\u7B20\uF9F8abc" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches("\uF9F8\uF9F8abc")) + assertTrue(regex.matches("\uF9F8\uF9F8abc")) // \u01F9 -> \u006E\u0300 // \u00C3 -> \u0041\u0303 @@ -856,7 +859,7 @@ fun testCanonEqFlag() { baseString = "cdb(?:\u00C3\u006E\u0300)" testString = "cdb\u0041\u0303\u01F9" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) // \u014C -> \u004F\u0304 // \u0163 -> \u0074\u0327 @@ -864,7 +867,7 @@ fun testCanonEqFlag() { baseString = "cdb(?:\u0163\u004F\u0304)" testString = "cdb\u0074\u0327\u014C" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) // \u00E1->a\u0301 // canonical ordering takes place \u0301\u0327 -> \u0327\u0301 @@ -872,7 +875,7 @@ fun testCanonEqFlag() { baseString = "c\u0327\u0301" testString = "c\u0301\u0327" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) /* * Hangul decompositions @@ -893,60 +896,60 @@ fun testCanonEqFlag() { baseString = "\uD4DB\uD21E\u1110\u1170cdb(ac)" testString = "\u1111\u1171\u11B6\u1110\u116D\u11B5\uD264cdbac" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\uD4DB\uD264cdb(a\uD21Ec)" testString = "\u1111\u1171\u11B6\u1110\u1170cdba\u1110\u116D\u11B5c" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "a\uD4DB" testString = "a\u1111\u1171\u11B6" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "a\uD21E" testString = "a\u1110\u116D\u11B5" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\u1111\u1171\u11B6cdb(ac\u1110\u116D\u11B5)" testString = "\uD4DBcdbac\uD21E" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "cdb(?:\u1111\u1171\u11B6\uD21E)" testString = "cdb\uD4DB\u1110\u116D\u11B5" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\uD4DB[a-c]\u1110\u116D\u11B5cdb(ac)" testString = "\uD4DBb\uD21Ecdbac" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\uD4DB|\u00CCcdb(ac)" testString = "\u1111\u1171\u11B6" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\uD4DB|\u00CCcdb(ac)" testString = "\u1111\u1171" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) baseString = "\u00CC?cdb(ac)*(\uD4DB)*[a-c]" testString = "cdb\u1111\u1171\u11B6b" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) baseString = "\uD4DB" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.containsMatchIn("a\u1111\u1171\u11B6a")) + assertTrue(regex.containsMatchIn("a\u1111\u1171\u11B6a")) baseString = "\u1111" regex = Regex(baseString, RegexOption.CANON_EQ) - Assert.assertFalse(regex.containsMatchIn("bcda\uD4DBr")) + assertFalse(regex.containsMatchIn("bcda\uD4DBr")) } fun testIndexesCanonicalEq() { @@ -959,25 +962,25 @@ fun testIndexesCanonicalEq() { testString = "bcda\u1111\u1171\u11B6awr" regex = Regex(baseString, RegexOption.CANON_EQ) result = regex.find(testString) - Assert.assertNotNull(result) - Assert.assertEquals(result!!.range.start, 4) - Assert.assertEquals(result.range.endInclusive, 6) + assertNotNull(result) + assertEquals(result!!.range.start, 4) + assertEquals(result.range.endInclusive, 6) baseString = "\uD4DB\u1111\u1171\u11B6" testString = "bcda\u1111\u1171\u11B6\uD4DBawr" regex = Regex(baseString, RegexOption.CANON_EQ) result = regex.find(testString) // Use the same testString - Assert.assertNotNull(result) - Assert.assertEquals(result!!.range.start, 4) - Assert.assertEquals(result.range.endInclusive, 7) + assertNotNull(result) + assertEquals(result!!.range.start, 4) + assertEquals(result.range.endInclusive, 7) baseString = "\uD4DB\uD21E\u1110\u1170" testString = "abcabc\u1111\u1171\u11B6\u1110\u116D\u11B5\uD264cdbac" regex = Regex(baseString, RegexOption.CANON_EQ) result = regex.find(testString) - Assert.assertNotNull(result) - Assert.assertEquals(result!!.range.start, 6) - Assert.assertEquals(result.range.endInclusive, 12) + assertNotNull(result) + assertEquals(result!!.range.start, 6) + assertEquals(result.range.endInclusive, 12) } fun testCanonEqFlagWithSupplementaryCharacters() { @@ -990,26 +993,26 @@ fun testCanonEqFlagWithSupplementaryCharacters() { var patString = "abc\uD834\uDDBFef" var testString = "abc\uD834\uDDB9\uD834\uDD65\uD834\uDD6Fef" var regex = Regex(patString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "abc\uD834\uDDBB\uD834\uDD6Fef" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) patString = "abc\uD834\uDDBB\uD834\uDD6Fef" testString = "abc\uD834\uDDBFef" regex = Regex(patString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "abc\uD834\uDDB9\uD834\uDD65\uD834\uDD6Fef" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) patString = "abc\uD834\uDDB9\uD834\uDD65\uD834\uDD6Fef" testString = "abc\uD834\uDDBFef" regex = Regex(patString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "abc\uD834\uDDBB\uD834\uDD6Fef" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) /* * testSupplementary characters with no decomposition @@ -1017,128 +1020,128 @@ fun testCanonEqFlagWithSupplementaryCharacters() { patString = "a\uD9A0\uDE8Ebc\uD834\uDDBB\uD834\uDD6Fe\uDE8Ef" testString = "a\uD9A0\uDE8Ebc\uD834\uDDBFe\uDE8Ef" regex = Regex(patString, RegexOption.CANON_EQ) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) } fun testRangesWithSurrogatesSupplementary() { var patString = "[abc\uD8D2]" var testString = "\uD8D2" var regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "a" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "ef\uD8D2\uDD71gh" - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "ef\uD8D2gh" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) patString = "[abc\uD8D3&&[c\uD8D3]]" testString = "c" regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "a" - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) testString = "ef\uD8D3\uDD71gh" - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "ef\uD8D3gh" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) patString = "[abc\uD8D3\uDBEE\uDF0C&&[c\uD8D3\uDBEE\uDF0C]]" testString = "c" regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "\uDBEE\uDF0C" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "ef\uD8D3\uDD71gh" - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "ef\uD8D3gh" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) patString = "[abc\uDBFC]\uDDC2cd" testString = "\uDBFC\uDDC2cd" regex = Regex(patString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) testString = "a\uDDC2cd" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) } fun testSequencesWithSurrogatesSupplementary() { var patString = "abcd\uD8D3" var testString = "abcd\uD8D3\uDFFC" var regex = Regex(patString) - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "abcd\uD8D3abc" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) patString = "ab\uDBEFcd" testString = "ab\uDBEFcd" regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) patString = "\uDFFCabcd" testString = "\uD8D3\uDFFCabcd" regex = Regex(patString) - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "abc\uDFFCabcdecd" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) patString = "\uD8D3\uDFFCabcd" testString = "abc\uD8D3\uD8D3\uDFFCabcd" regex = Regex(patString) - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) } fun testPredefinedClassesWithSurrogatesSupplementary() { var patString = "[123\\D]" var testString = "a" var regex = Regex(patString) - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) testString = "5" - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "3" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) // low surrogate testString = "\uDFC4" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) // high surrogate testString = "\uDADA" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) testString = "\uDADA\uDFC4" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) testString = "5" - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "3" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) // low surrogate testString = "\uDFC4" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) // high surrogate testString = "\uDADA" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) testString = "\uDADA\uDFC4" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) // surrogate characters patString = "\\p{Cs}" @@ -1151,86 +1154,86 @@ fun testPredefinedClassesWithSurrogatesSupplementary() { * consisting of two code units (two surrogate characters) so we find * nothing */ - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) // swap low and high surrogates testString = "\uDE27\uD916" - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) patString = "[\uD916\uDE271\uD91623&&[^\\p{Cs}]]" testString = "1" regex = Regex(patString) - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) testString = "\uD916" regex = Regex(patString) - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) testString = "\uD916\uDE27" regex = Regex(patString) - Assert.assertTrue(regex.containsMatchIn(testString)) + assertTrue(regex.containsMatchIn(testString)) // \uD9A0\uDE8E=\u7828E // \u78281=\uD9A0\uDE81 patString = "[a-\uD9A0\uDE8E]" testString = "\uD9A0\uDE81" regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) } fun testDotConstructionWithSurrogatesSupplementary() { var patString = "." var testString = "\uD9A0\uDE81" var regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "\uDE81" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "\uD9A0" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "\n" - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) patString = ".*\uDE81" testString = "\uD9A0\uDE81\uD9A0\uDE81\uD9A0\uDE81" regex = Regex(patString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) testString = "\uD9A0\uDE81\uD9A0\uDE81\uDE81" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) patString = ".*" testString = "\uD9A0\uDE81\n\uD9A0\uDE81\uD9A0\n\uDE81" regex = Regex(patString, RegexOption.DOT_MATCHES_ALL) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) } fun testQuantifiersWithSurrogatesSupplementary() { val patString = "\uD9A0\uDE81*abc" var testString = "\uD9A0\uDE81\uD9A0\uDE81abc" val regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "abc" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) } fun testAlternationsWithSurrogatesSupplementary() { val patString = "\uDE81|\uD9A0\uDE81|\uD9A0" var testString = "\uD9A0" val regex = Regex(patString) - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "\uDE81" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "\uD9A0\uDE81" - Assert.assertTrue(regex.matches(testString)) + assertTrue(regex.matches(testString)) testString = "\uDE81\uD9A0" - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) } fun testGroupsWithSurrogatesSupplementary() { @@ -1239,12 +1242,12 @@ fun testGroupsWithSurrogatesSupplementary() { var patString = "(\uD9A0)\uDE81" var testString = "\uD9A0\uDE81" var regex = Regex(patString) - Assert.assertFalse(regex.matches(testString)) + assertFalse(regex.matches(testString)) patString = "(\uD9A0)" testString = "\uD9A0\uDE81" regex = Regex(patString, RegexOption.DOT_MATCHES_ALL) - Assert.assertFalse(regex.containsMatchIn(testString)) + assertFalse(regex.containsMatchIn(testString)) } fun box() { diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest2.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest2.kt index 35e54b09cee..903a21bbe44 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest2.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/PatternTest2.kt @@ -17,6 +17,9 @@ import kotlin.text.* import kotlin.test.* +fun assertTrue(msg: String, value: Boolean) = assertTrue(value, msg) +fun assertFalse(msg: String, value: Boolean) = assertFalse(value, msg) + /** * Tests simple pattern compilation and matching methods */ @@ -24,19 +27,19 @@ fun testSimpleMatch() { val regex = Regex("foo.*") var testString = "foo123" - Assert.assertTrue(regex.matches(testString)) - Assert.assertTrue(regex.containsMatchIn(testString, 0)) - Assert.assertTrue(regex.lookingAt(testString)) + assertTrue(regex.matches(testString)) + assertTrue(regex.containsMatchIn(testString, 0)) + assertTrue(regex.lookingAt(testString)) testString = "fox" - Assert.assertFalse(regex.matches(testString)) - Assert.assertFalse(regex.containsMatchIn(testString, 0)) - Assert.assertFalse(regex.lookingAt(testString)) + assertFalse(regex.matches(testString)) + assertFalse(regex.containsMatchIn(testString, 0)) + assertFalse(regex.lookingAt(testString)) - Assert.assertTrue(Regex("foo.*").matches("foo123")) - Assert.assertFalse(Regex("foo.*").matches("fox")) - Assert.assertFalse(Regex("bar").matches("foobar")) - Assert.assertTrue(Regex("").matches("")) + assertTrue(Regex("foo.*").matches("foo123")) + assertFalse(Regex("foo.*").matches("fox")) + assertFalse(Regex("bar").matches("foobar")) + assertTrue(Regex("").matches("")) } fun testCursors() { @@ -47,50 +50,50 @@ fun testCursors() { regex = Regex("foo") result = regex.find("foobar") - Assert.assertNotNull(result) - Assert.assertEquals(0, result!!.range.start) - Assert.assertEquals(3, result.range.endInclusive + 1) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals(0, result!!.range.start) + assertEquals(3, result.range.endInclusive + 1) + assertNull(result.next()) result = regex.find("barfoobar") - Assert.assertNotNull(result) - Assert.assertEquals(3, result!!.range.start) - Assert.assertEquals(6, result.range.endInclusive + 1) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals(3, result!!.range.start) + assertEquals(6, result.range.endInclusive + 1) + assertNull(result.next()) result = regex.find("barfoo") - Assert.assertNotNull(result) - Assert.assertEquals(3, result!!.range.start) - Assert.assertEquals(6, result.range.endInclusive + 1) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals(3, result!!.range.start) + assertEquals(6, result.range.endInclusive + 1) + assertNull(result.next()) result = regex.find("foobarfoobarfoo") - Assert.assertNotNull(result) - Assert.assertEquals(0, result!!.range.start) - Assert.assertEquals(3, result.range.endInclusive + 1) + assertNotNull(result) + assertEquals(0, result!!.range.start) + assertEquals(3, result.range.endInclusive + 1) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals(6, result!!.range.start) - Assert.assertEquals(9, result.range.endInclusive + 1) + assertNotNull(result) + assertEquals(6, result!!.range.start) + assertEquals(9, result.range.endInclusive + 1) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals(12, result!!.range.start) - Assert.assertEquals(15, result.range.endInclusive + 1) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals(12, result!!.range.start) + assertEquals(15, result.range.endInclusive + 1) + assertNull(result.next()) result = regex.find("foobarfoobarfoo", 0) - Assert.assertNotNull(result) - Assert.assertEquals(0, result!!.range.start) - Assert.assertEquals(3, result.range.endInclusive + 1) + assertNotNull(result) + assertEquals(0, result!!.range.start) + assertEquals(3, result.range.endInclusive + 1) result = regex.find("foobarfoobarfoo", 4) - Assert.assertNotNull(result) - Assert.assertEquals(6, result!!.range.start) - Assert.assertEquals(9, result.range.endInclusive + 1) + assertNotNull(result) + assertEquals(6, result!!.range.start) + assertEquals(9, result.range.endInclusive + 1) } catch (e: PatternSyntaxException) { println(e.message) - Assert.fail() + fail() } } @@ -102,52 +105,52 @@ fun testGroups() { regex = Regex("(p[0-9]*)#?(q[0-9]*)") result = regex.find("p1#q3p2q42p5p71p63#q888") - Assert.assertNotNull(result) - Assert.assertEquals(0, result!!.range.start) - Assert.assertEquals(5, result.range.endInclusive + 1) - Assert.assertEquals(3, result.groups.size) - Assert.assertEquals(0, result.groups[0]!!.range.start) - Assert.assertEquals(5, result.groups[0]!!.range.endInclusive + 1) - Assert.assertEquals(0, result.groups[1]!!.range.start) - Assert.assertEquals(2, result.groups[1]!!.range.endInclusive + 1) - Assert.assertEquals(3, result.groups[2]!!.range.start) - Assert.assertEquals(5, result.groups[2]!!.range.endInclusive + 1) - Assert.assertEquals("p1#q3", result.value) - Assert.assertEquals("p1#q3", result.groupValues[0]) - Assert.assertEquals("p1", result.groupValues[1]) - Assert.assertEquals("q3", result.groupValues[2]) + assertNotNull(result) + assertEquals(0, result!!.range.start) + assertEquals(5, result.range.endInclusive + 1) + assertEquals(3, result.groups.size) + assertEquals(0, result.groups[0]!!.range.start) + assertEquals(5, result.groups[0]!!.range.endInclusive + 1) + assertEquals(0, result.groups[1]!!.range.start) + assertEquals(2, result.groups[1]!!.range.endInclusive + 1) + assertEquals(3, result.groups[2]!!.range.start) + assertEquals(5, result.groups[2]!!.range.endInclusive + 1) + assertEquals("p1#q3", result.value) + assertEquals("p1#q3", result.groupValues[0]) + assertEquals("p1", result.groupValues[1]) + assertEquals("q3", result.groupValues[2]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals(5, result!!.range.start) - Assert.assertEquals(10, result.range.endInclusive + 1) - Assert.assertEquals(3, result.groups.size) - Assert.assertEquals(10, result.groups[0]!!.range.endInclusive + 1) - Assert.assertEquals(5, result.groups[1]!!.range.start) - Assert.assertEquals(7, result.groups[1]!!.range.endInclusive + 1) - Assert.assertEquals(7, result.groups[2]!!.range.start) - Assert.assertEquals(10, result.groups[2]!!.range.endInclusive + 1) - Assert.assertEquals("p2q42", result.value) - Assert.assertEquals("p2q42", result.groupValues[0]) - Assert.assertEquals("p2", result.groupValues[1]) - Assert.assertEquals("q42", result.groupValues[2]) + assertNotNull(result) + assertEquals(5, result!!.range.start) + assertEquals(10, result.range.endInclusive + 1) + assertEquals(3, result.groups.size) + assertEquals(10, result.groups[0]!!.range.endInclusive + 1) + assertEquals(5, result.groups[1]!!.range.start) + assertEquals(7, result.groups[1]!!.range.endInclusive + 1) + assertEquals(7, result.groups[2]!!.range.start) + assertEquals(10, result.groups[2]!!.range.endInclusive + 1) + assertEquals("p2q42", result.value) + assertEquals("p2q42", result.groupValues[0]) + assertEquals("p2", result.groupValues[1]) + assertEquals("q42", result.groupValues[2]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals(15, result!!.range.start) - Assert.assertEquals(23, result.range.endInclusive + 1) - Assert.assertEquals(3, result.groups.size) - Assert.assertEquals(15, result.groups[0]!!.range.start) - Assert.assertEquals(23, result.groups[0]!!.range.endInclusive + 1) - Assert.assertEquals(15, result.groups[1]!!.range.start) - Assert.assertEquals(18, result.groups[1]!!.range.endInclusive + 1) - Assert.assertEquals(19, result.groups[2]!!.range.start) - Assert.assertEquals(23, result.groups[2]!!.range.endInclusive + 1) - Assert.assertEquals("p63#q888", result.value) - Assert.assertEquals("p63#q888", result.groupValues[0]) - Assert.assertEquals("p63", result.groupValues[1]) - Assert.assertEquals("q888", result.groupValues[2]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals(15, result!!.range.start) + assertEquals(23, result.range.endInclusive + 1) + assertEquals(3, result.groups.size) + assertEquals(15, result.groups[0]!!.range.start) + assertEquals(23, result.groups[0]!!.range.endInclusive + 1) + assertEquals(15, result.groups[1]!!.range.start) + assertEquals(18, result.groups[1]!!.range.endInclusive + 1) + assertEquals(19, result.groups[2]!!.range.start) + assertEquals(23, result.groups[2]!!.range.endInclusive + 1) + assertEquals("p63#q888", result.value) + assertEquals("p63#q888", result.groupValues[0]) + assertEquals("p63", result.groupValues[1]) + assertEquals("q888", result.groupValues[2]) + assertNull(result.next()) } fun testReplace() { @@ -158,32 +161,32 @@ fun testReplace() { regex = Regex("a*b") var testString = "aabfooaabfooabfoob" - Assert.assertTrue(regex.replace(testString, "-") == "-foo-foo-foo-") - Assert.assertTrue(regex.replaceFirst(testString, "-") == "-fooaabfooabfoob") + assertTrue(regex.replace(testString, "-") == "-foo-foo-foo-") + assertTrue(regex.replaceFirst(testString, "-") == "-fooaabfooabfoob") regex = Regex("([bB])yte") testString = "Byte for byte" - Assert.assertTrue(regex.replaceFirst(testString, "$1ite") == "Bite for byte") - Assert.assertTrue(regex.replace(testString, "$1ite") == "Bite for bite") + assertTrue(regex.replaceFirst(testString, "$1ite") == "Bite for byte") + assertTrue(regex.replace(testString, "$1ite") == "Bite for bite") regex = Regex("\\d\\d\\d\\d([- ])") testString = "card #1234-5678-1234" - Assert.assertTrue(regex.replaceFirst(testString, "xxxx$1") == "card #xxxx-5678-1234") - Assert.assertTrue(regex.replace(testString, "xxxx$1") == "card #xxxx-xxxx-1234") + assertTrue(regex.replaceFirst(testString, "xxxx$1") == "card #xxxx-5678-1234") + assertTrue(regex.replace(testString, "xxxx$1") == "card #xxxx-xxxx-1234") regex = Regex("(up|left)( *)(right|down)") testString = "left right, up down" - Assert.assertTrue(regex.replaceFirst(testString, "$3$2$1") == "right left, up down") - Assert.assertTrue(regex.replace(testString, "$3$2$1") == "right left, down up") + assertTrue(regex.replaceFirst(testString, "$3$2$1") == "right left, up down") + assertTrue(regex.replace(testString, "$3$2$1") == "right left, down up") regex = Regex("([CcPp][hl]e[ea]se)") testString = "I want cheese. Please." - Assert.assertTrue(regex.replaceFirst(testString, " $1 ") == "I want cheese . Please.") - Assert.assertTrue(regex.replace(testString, " $1 ") == "I want cheese . Please .") + assertTrue(regex.replaceFirst(testString, " $1 ") == "I want cheese . Please.") + assertTrue(regex.replace(testString, " $1 ") == "I want cheese . Please .") } fun testEscapes() { @@ -193,149 +196,149 @@ fun testEscapes() { // Test \\ sequence regex = Regex("([a-z]+)\\\\([a-z]+);") result = regex.find("fred\\ginger;abbott\\costello;jekell\\hyde;") - Assert.assertNotNull(result) - Assert.assertEquals("fred", result!!.groupValues[1]) - Assert.assertEquals("ginger", result.groupValues[2]) + assertNotNull(result) + assertEquals("fred", result!!.groupValues[1]) + assertEquals("ginger", result.groupValues[2]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("abbott", result!!.groupValues[1]) - Assert.assertEquals("costello", result.groupValues[2]) + assertNotNull(result) + assertEquals("abbott", result!!.groupValues[1]) + assertEquals("costello", result.groupValues[2]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("jekell", result!!.groupValues[1]) - Assert.assertEquals("hyde", result.groupValues[2]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("jekell", result!!.groupValues[1]) + assertEquals("hyde", result.groupValues[2]) + assertNull(result.next()) // Test \n, \t, \r, \f, \e, \a sequences regex = Regex("([a-z]+)[\\n\\t\\r\\f\\e\\a]+([a-z]+)") result = regex.find("aa\nbb;cc\u0009\rdd;ee\u000C\u001Bff;gg\n\u0007hh") - Assert.assertNotNull(result) - Assert.assertEquals("aa", result!!.groupValues[1]) - Assert.assertEquals("bb", result.groupValues[2]) + assertNotNull(result) + assertEquals("aa", result!!.groupValues[1]) + assertEquals("bb", result.groupValues[2]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("cc", result!!.groupValues[1]) - Assert.assertEquals("dd", result.groupValues[2]) + assertNotNull(result) + assertEquals("cc", result!!.groupValues[1]) + assertEquals("dd", result.groupValues[2]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("ee", result!!.groupValues[1]) - Assert.assertEquals("ff", result.groupValues[2]) + assertNotNull(result) + assertEquals("ee", result!!.groupValues[1]) + assertEquals("ff", result.groupValues[2]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("gg", result!!.groupValues[1]) - Assert.assertEquals("hh", result.groupValues[2]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("gg", result!!.groupValues[1]) + assertEquals("hh", result.groupValues[2]) + assertNull(result.next()) // Test \\u and \\x sequences regex = Regex("([0-9]+)[\\u0020:\\x21];") result = regex.find("11:;22 ;33-;44!;") - Assert.assertNotNull(result) - Assert.assertEquals("11", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("11", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("22", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("22", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("44", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("44", result!!.groupValues[1]) + assertNull(result.next()) // Test invalid unicode sequences // TODO: Double check it. try { regex = Regex("\\u") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\u;") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\u002") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\u002;") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } // Test invalid hex sequences try { regex = Regex("\\x") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\x;") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\xa") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\xa;") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } // Test \0 (octal) sequences (1, 2 and 3 digit) regex = Regex("([0-9]+)[\\07\\040\\0160];") result = regex.find("11\u0007;22:;33 ;44p;") - Assert.assertNotNull(result) - Assert.assertEquals("11", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("11", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("33", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("33", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("44", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("44", result!!.groupValues[1]) + assertNull(result.next()) // Test invalid octal sequences try { regex = Regex("\\08") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\0") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } try { regex = Regex("\\0;") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } // Test \c (control character) sequence regex = Regex("([0-9]+)[\\cA\\cB\\cC\\cD];") result = regex.find("11\u0001;22:;33\u0002;44p;55\u0003;66\u0004;") - Assert.assertNotNull(result) - Assert.assertEquals("11", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("11", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("33", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("33", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("55", result!!.groupValues[1]) + assertNotNull(result) + assertEquals("55", result!!.groupValues[1]) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("66", result!!.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("66", result!!.groupValues[1]) + assertNull(result.next()) // More thorough control escape test // Ensure that each escape matches exactly the corresponding @@ -346,18 +349,18 @@ fun testEscapes() { var match_char = -1 for (j in 0..255) { if (regex.matches("${j.toChar()}")) { - Assert.assertEquals(-1, match_char) + assertEquals(-1, match_char) match_char = j } } - Assert.assertTrue(match_char == i + 1) + assertTrue(match_char == i + 1) } // Test invalid control escapes try { regex = Regex("\\c") - Assert.fail("PatternSyntaxException expected") + fail("PatternSyntaxException expected") } catch (e: PatternSyntaxException) { } } @@ -367,69 +370,69 @@ fun testCharacterClasses() { // Test one character range regex = Regex("[p].*[l]") - Assert.assertTrue(regex.matches("paul")) - Assert.assertTrue(regex.matches("pool")) - Assert.assertFalse(regex.matches("pong")) - Assert.assertTrue(regex.matches("pl")) + assertTrue(regex.matches("paul")) + assertTrue(regex.matches("pool")) + assertFalse(regex.matches("pong")) + assertTrue(regex.matches("pl")) // Test two character range regex = Regex("[pm].*[lp]") - Assert.assertTrue(regex.matches("prop")) - Assert.assertTrue(regex.matches("mall")) - Assert.assertFalse(regex.matches("pong")) - Assert.assertTrue(regex.matches("pill")) + assertTrue(regex.matches("prop")) + assertTrue(regex.matches("mall")) + assertFalse(regex.matches("pong")) + assertTrue(regex.matches("pill")) // Test range including [ and ] regex = Regex("[<\\[].*[\\]>]") - Assert.assertTrue(regex.matches("")) - Assert.assertTrue(regex.matches("[bar]")) - Assert.assertFalse(regex.matches("{foobar]")) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("[bar]")) + assertFalse(regex.matches("{foobar]")) + assertTrue(regex.matches("") - Assert.assertTrue(regex.matches("")) - Assert.assertFalse(regex.matches("")) + assertTrue(regex.matches("")) + assertFalse(regex.matches("")) val result = regex.find("xyz zzz") - Assert.assertNotNull(result) - Assert.assertNotNull(result!!.next()) - Assert.assertNull(result.next()!!.next()) + assertNotNull(result) + assertNotNull(result!!.next()) + assertNull(result.next()!!.next()) // Test \S (not whitespace) // TODO: We've removed \f from string since kotlin doesn't recognize this escape in a string. regex = Regex("<[a-z] \\S[0-9][\\S\n]+[^\\S]221>") - Assert.assertTrue(regex.matches("")) - Assert.assertTrue(regex.matches("")) - Assert.assertFalse(regex.matches("")) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) + assertTrue(regex.matches("")) + assertFalse(regex.matches("")) + assertTrue(regex.matches("")) regex = Regex("<[a-z] \\S[0-9][\\S\n]+[^\\S]221[\\S&&[^abc]]>") - Assert.assertTrue(regex.matches("")) - Assert.assertTrue(regex.matches("")) - Assert.assertFalse(regex.matches("")) - Assert.assertFalse(regex.matches("")) - Assert.assertFalse(regex.matches("")) - Assert.assertTrue(regex.matches("")) + assertTrue(regex.matches("")) + assertTrue(regex.matches("")) + assertFalse(regex.matches("")) + assertFalse(regex.matches("")) + assertFalse(regex.matches("")) + assertTrue(regex.matches("")) // Test \w (ascii word) regex = Regex("<\\w+\\s[0-9]+;[^\\w]\\w+/[\\w$]+;") - Assert.assertTrue(regex.matches(" 0) { - Assert.assertFalse(regex.matches((block.low - 1).toChar().toString())) + assertFalse(regex.matches((block.low - 1).toChar().toString())) } for (i in block.low..block.high) { - Assert.assertTrue(regex.matches(i.toChar().toString())) + assertTrue(regex.matches(i.toChar().toString())) } if (block.high < 0xFFFF) { - Assert.assertFalse(regex.matches((block.high + 1).toChar().toString())) + assertFalse(regex.matches((block.high + 1).toChar().toString())) } regex = Regex("\\P{In" + block.name + "}") if (block.low > 0) { - Assert.assertTrue(regex.matches((block.low - 1).toChar().toString())) + assertTrue(regex.matches((block.low - 1).toChar().toString())) } for (i in block.low..block.high) { - Assert.assertFalse("assert: Regex: $regex, match to: ${i.toChar()} ($i)", regex.matches(i.toChar().toString())) + assertFalse("assert: Regex: $regex, match to: ${i.toChar()} ($i)", regex.matches(i.toChar().toString())) } if (block.high < 0xFFFF) { - Assert.assertTrue(regex.matches((block.high + 1).toChar().toString())) + assertTrue(regex.matches((block.high + 1).toChar().toString())) } } @@ -838,27 +841,27 @@ fun testMisc() { // Test \Q, \E regex = Regex("[a-z]+;\\Q[a-z]+;\\Q(foo.*);\\E[0-9]+") - Assert.assertTrue(regex.matches("abc;[a-z]+;\\Q(foo.*);411")) - Assert.assertFalse(regex.matches("abc;def;foo42;555")) - Assert.assertFalse(regex.matches("abc;\\Qdef;\\Qfoo99;\\E123")) + assertTrue(regex.matches("abc;[a-z]+;\\Q(foo.*);411")) + assertFalse(regex.matches("abc;def;foo42;555")) + assertFalse(regex.matches("abc;\\Qdef;\\Qfoo99;\\E123")) regex = Regex("[a-z]+;(foo[0-9]-\\Q(...)\\E);[0-9]+") val result = regex.matchEntire("abc;foo5-(...);123") - Assert.assertNotNull(result) - Assert.assertEquals("foo5-(...)", result!!.groupValues[1]) - Assert.assertFalse(regex.matches("abc;foo9-(xxx);789")) + assertNotNull(result) + assertEquals("foo5-(...)", result!!.groupValues[1]) + assertFalse(regex.matches("abc;foo9-(xxx);789")) regex = Regex("[a-z]+;(bar[0-9]-[a-z\\Q$-\\E]+);[0-9]+") - Assert.assertTrue(regex.matches("abc;bar0-def$-;123")) + assertTrue(regex.matches("abc;bar0-def$-;123")) regex = Regex("[a-z]+;(bar[0-9]-[a-z\\Q-$\\E]+);[0-9]+") - Assert.assertTrue(regex.matches("abc;bar0-def$-;123")) + assertTrue(regex.matches("abc;bar0-def$-;123")) regex = Regex("[a-z]+;(bar[0-9]-[a-z\\Q[0-9]\\E]+);[0-9]+") - Assert.assertTrue(regex.matches("abc;bar0-def[99]-]0x[;123")); + assertTrue(regex.matches("abc;bar0-def[99]-]0x[;123")); regex = Regex("[a-z]+;(bar[0-9]-[a-z\\[0\\-9\\]]+);[0-9]+") - Assert.assertTrue(regex.matches("abc;bar0-def[99]-]0x[;123")) + assertTrue(regex.matches("abc;bar0-def[99]-]0x[;123")) // Test # // TODO @@ -867,7 +870,7 @@ fun testMisc() { fun testCompile1() { val regex = Regex("[0-9A-Za-z][0-9A-Za-z\\x2e\\x3a\\x2d\\x5f]*") val name = "iso-8859-1" - Assert.assertTrue(regex.matches(name)) + assertTrue(regex.matches(name)) } fun testCompile2() { @@ -882,313 +885,310 @@ fun testCompile3() { regex = Regex("a$") result = regex.find("a\n") - Assert.assertNotNull(result) - Assert.assertEquals("a", result!!.value) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("a", result!!.value) + assertNull(result.next()) regex = Regex("(a$)") result = regex.find("a\n") - Assert.assertNotNull(result) - Assert.assertEquals("a", result!!.value) - Assert.assertEquals("a", result.groupValues[1]) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("a", result!!.value) + assertEquals("a", result.groupValues[1]) + assertNull(result.next()) regex = Regex("^.*$", RegexOption.MULTILINE) result = regex.find("a\n") - Assert.assertNotNull(result) - Assert.assertEquals("a", result!!.value) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("a", result!!.value) + assertNull(result.next()) result = regex.find("a\nb\n") - Assert.assertNotNull(result) - Assert.assertEquals("a", result!!.value) + assertNotNull(result) + assertEquals("a", result!!.value) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("b", result!!.value) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("b", result!!.value) + assertNull(result.next()) result = regex.find("a\nb") - Assert.assertNotNull(result) - Assert.assertEquals("a", result!!.value) + assertNotNull(result) + assertEquals("a", result!!.value) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("b", result!!.value) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("b", result!!.value) + assertNull(result.next()) result = regex.find("\naa\r\nbb\rcc\n\n") - Assert.assertNotNull(result) - Assert.assertTrue(result!!.value == "") + assertNotNull(result) + assertTrue(result!!.value == "") result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("aa", result!!.value) + assertNotNull(result) + assertEquals("aa", result!!.value) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("bb", result!!.value) + assertNotNull(result) + assertEquals("bb", result!!.value) result = result.next() - Assert.assertNotNull(result) - Assert.assertEquals("cc", result!!.value) + assertNotNull(result) + assertEquals("cc", result!!.value) result = result.next() - Assert.assertNotNull(result) - Assert.assertTrue(result!!.value == "") - Assert.assertNull(result.next()) + assertNotNull(result) + assertTrue(result!!.value == "") + assertNull(result.next()) result = regex.find("a") - Assert.assertNotNull(result) - Assert.assertEquals("a", result!!.value) - Assert.assertNull(result.next()) + assertNotNull(result) + assertEquals("a", result!!.value) + assertNull(result.next()) result = regex.find("") - Assert.assertNull(result) + assertNull(result) regex = Regex("^.*$") result = regex.find("") - Assert.assertNotNull(result) - Assert.assertTrue(result!!.value == "") - Assert.assertNull(result.next()) + assertNotNull(result) + assertTrue(result!!.value == "") + assertNull(result.next()) } fun testCompile4() { val findString = "\\Qpublic\\E" - val text = StringBuffer(" public class Class {\n" + " public class Class {") + val text = StringBuilder(" public class Class {\n" + " public class Class {") val regex = Regex(findString) val result = regex.find(text) - Assert.assertNotNull(result) - Assert.assertEquals(4, result!!.range.start) + assertNotNull(result) + assertEquals(4, result!!.range.start) // modify text - text.delete(0, text.length) + text.length = 0 text.append("Text have been changed.") - // TODO: Check the existing result. - Assert.assertNull(regex.find(text)) + assertNull(regex.find(text)) } fun testCompile5() { val p = Regex("^[0-9]") val s = p.split("12", 0) - Assert.assertEquals("", s[0]) - Assert.assertEquals("2", s[1]) - Assert.assertEquals(2, s.size) + assertEquals("", s[0]) + assertEquals("2", s[1]) + assertEquals(2, s.size) } private class UBInfo(var low: Int, var high: Int, var name: String) -companion object { +// A table representing the unicode categories +// private static UBInfo[] UCategories = { +// Lu +// Ll +// Lt +// Lm +// Lo +// Mn +// Mc +// Me +// Nd +// Nl +// No +// Pc +// Pd +// Ps +// Pe +// Pi +// Pf +// Po +// Sm +// Sc +// Sk +// So +// Zs +// Zl +// Zp +// Cc +// Cf +// Cs +// Co +// Cn +// }; - // A table representing the unicode categories - // private static UBInfo[] UCategories = { - // Lu - // Ll - // Lt - // Lm - // Lo - // Mn - // Mc - // Me - // Nd - // Nl - // No - // Pc - // Pd - // Ps - // Pe - // Pi - // Pf - // Po - // Sm - // Sc - // Sk - // So - // Zs - // Zl - // Zp - // Cc - // Cf - // Cs - // Co - // Cn - // }; +// A table representing the unicode character blocks +private val UBlocks = arrayOf( + /* 0000; 007F; Basic Latin */ + UBInfo(0x0000, 0x007F, "BasicLatin"), // Character.UnicodeBlock.BASIC_LATIN + /* 0080; 00FF; Latin-1 Supplement */ + UBInfo(0x0080, 0x00FF, "Latin-1Supplement"), // Character.UnicodeBlock.LATIN_1_SUPPLEMENT + /* 0100; 017F; Latin Extended-A */ + UBInfo(0x0100, 0x017F, "LatinExtended-A"), // Character.UnicodeBlock.LATIN_EXTENDED_A + /* 0180; 024F; Latin Extended-B */ + // new UBInfo (0x0180,0x024F,"InLatinExtended-B"), // + // Character.UnicodeBlock.LATIN_EXTENDED_B + /* 0250; 02AF; IPA Extensions */ + UBInfo(0x0250, 0x02AF, "IPAExtensions"), // Character.UnicodeBlock.IPA_EXTENSIONS + /* 02B0; 02FF; Spacing Modifier Letters */ + UBInfo(0x02B0, 0x02FF, "SpacingModifierLetters"), // Character.UnicodeBlock.SPACING_MODIFIER_LETTERS + /* 0300; 036F; Combining Diacritical Marks */ + UBInfo(0x0300, 0x036F, "CombiningDiacriticalMarks"), // Character.UnicodeBlock.COMBINING_DIACRITICAL_MARKS + /* 0370; 03FF; Greek */ + UBInfo(0x0370, 0x03FF, "Greek"), // Character.UnicodeBlock.GREEK + /* 0400; 04FF; Cyrillic */ + UBInfo(0x0400, 0x04FF, "Cyrillic"), // Character.UnicodeBlock.CYRILLIC + /* 0530; 058F; Armenian */ + UBInfo(0x0530, 0x058F, "Armenian"), // Character.UnicodeBlock.ARMENIAN + /* 0590; 05FF; Hebrew */ + UBInfo(0x0590, 0x05FF, "Hebrew"), // Character.UnicodeBlock.HEBREW + /* 0600; 06FF; Arabic */ + UBInfo(0x0600, 0x06FF, "Arabic"), // Character.UnicodeBlock.ARABIC + /* 0700; 074F; Syriac */ + UBInfo(0x0700, 0x074F, "Syriac"), // Character.UnicodeBlock.SYRIAC + /* 0780; 07BF; Thaana */ + UBInfo(0x0780, 0x07BF, "Thaana"), // Character.UnicodeBlock.THAANA + /* 0900; 097F; Devanagari */ + UBInfo(0x0900, 0x097F, "Devanagari"), // Character.UnicodeBlock.DEVANAGARI + /* 0980; 09FF; Bengali */ + UBInfo(0x0980, 0x09FF, "Bengali"), // Character.UnicodeBlock.BENGALI + /* 0A00; 0A7F; Gurmukhi */ + UBInfo(0x0A00, 0x0A7F, "Gurmukhi"), // Character.UnicodeBlock.GURMUKHI + /* 0A80; 0AFF; Gujarati */ + UBInfo(0x0A80, 0x0AFF, "Gujarati"), // Character.UnicodeBlock.GUJARATI + /* 0B00; 0B7F; Oriya */ + UBInfo(0x0B00, 0x0B7F, "Oriya"), // Character.UnicodeBlock.ORIYA + /* 0B80; 0BFF; Tamil */ + UBInfo(0x0B80, 0x0BFF, "Tamil"), // Character.UnicodeBlock.TAMIL + /* 0C00; 0C7F; Telugu */ + UBInfo(0x0C00, 0x0C7F, "Telugu"), // Character.UnicodeBlock.TELUGU + /* 0C80; 0CFF; Kannada */ + UBInfo(0x0C80, 0x0CFF, "Kannada"), // Character.UnicodeBlock.KANNADA + /* 0D00; 0D7F; Malayalam */ + UBInfo(0x0D00, 0x0D7F, "Malayalam"), // Character.UnicodeBlock.MALAYALAM + /* 0D80; 0DFF; Sinhala */ + UBInfo(0x0D80, 0x0DFF, "Sinhala"), // Character.UnicodeBlock.SINHALA + /* 0E00; 0E7F; Thai */ + UBInfo(0x0E00, 0x0E7F, "Thai"), // Character.UnicodeBlock.THAI + /* 0E80; 0EFF; Lao */ + UBInfo(0x0E80, 0x0EFF, "Lao"), // Character.UnicodeBlock.LAO + /* 0F00; 0FFF; Tibetan */ + UBInfo(0x0F00, 0x0FFF, "Tibetan"), // Character.UnicodeBlock.TIBETAN + /* 1000; 109F; Myanmar */ + UBInfo(0x1000, 0x109F, "Myanmar"), // Character.UnicodeBlock.MYANMAR + /* 10A0; 10FF; Georgian */ + UBInfo(0x10A0, 0x10FF, "Georgian"), // Character.UnicodeBlock.GEORGIAN + /* 1100; 11FF; Hangul Jamo */ + UBInfo(0x1100, 0x11FF, "HangulJamo"), // Character.UnicodeBlock.HANGUL_JAMO + /* 1200; 137F; Ethiopic */ + UBInfo(0x1200, 0x137F, "Ethiopic"), // Character.UnicodeBlock.ETHIOPIC + /* 13A0; 13FF; Cherokee */ + UBInfo(0x13A0, 0x13FF, "Cherokee"), // Character.UnicodeBlock.CHEROKEE + /* 1400; 167F; Unified Canadian Aboriginal Syllabics */ + UBInfo(0x1400, 0x167F, "UnifiedCanadianAboriginalSyllabics"), // Character.UnicodeBlock.UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS + /* 1680; 169F; Ogham */ + UBInfo(0x1680, 0x169F, "Ogham"), // Character.UnicodeBlock.OGHAM + /* 16A0; 16FF; Runic */ + UBInfo(0x16A0, 0x16FF, "Runic"), // Character.UnicodeBlock.RUNIC + /* 1780; 17FF; Khmer */ + UBInfo(0x1780, 0x17FF, "Khmer"), // Character.UnicodeBlock.KHMER + /* 1800; 18AF; Mongolian */ + UBInfo(0x1800, 0x18AF, "Mongolian"), // Character.UnicodeBlock.MONGOLIAN + /* 1E00; 1EFF; Latin Extended Additional */ + UBInfo(0x1E00, 0x1EFF, "LatinExtendedAdditional"), // Character.UnicodeBlock.LATIN_EXTENDED_ADDITIONAL + /* 1F00; 1FFF; Greek Extended */ + UBInfo(0x1F00, 0x1FFF, "GreekExtended"), // Character.UnicodeBlock.GREEK_EXTENDED + /* 2000; 206F; General Punctuation */ + UBInfo(0x2000, 0x206F, "GeneralPunctuation"), // Character.UnicodeBlock.GENERAL_PUNCTUATION + /* 2070; 209F; Superscripts and Subscripts */ + UBInfo(0x2070, 0x209F, "SuperscriptsandSubscripts"), // Character.UnicodeBlock.SUPERSCRIPTS_AND_SUBSCRIPTS + /* 20A0; 20CF; Currency Symbols */ + UBInfo(0x20A0, 0x20CF, "CurrencySymbols"), // Character.UnicodeBlock.CURRENCY_SYMBOLS + /* 20D0; 20FF; Combining Marks for Symbols */ + UBInfo(0x20D0, 0x20FF, "CombiningMarksforSymbols"), // Character.UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS + /* 2100; 214F; Letterlike Symbols */ + UBInfo(0x2100, 0x214F, "LetterlikeSymbols"), // Character.UnicodeBlock.LETTERLIKE_SYMBOLS + /* 2150; 218F; Number Forms */ + UBInfo(0x2150, 0x218F, "NumberForms"), // Character.UnicodeBlock.NUMBER_FORMS + /* 2190; 21FF; Arrows */ + UBInfo(0x2190, 0x21FF, "Arrows"), // Character.UnicodeBlock.ARROWS + /* 2200; 22FF; Mathematical Operators */ + UBInfo(0x2200, 0x22FF, "MathematicalOperators"), // Character.UnicodeBlock.MATHEMATICAL_OPERATORS + /* 2300; 23FF; Miscellaneous Technical */ + UBInfo(0x2300, 0x23FF, "MiscellaneousTechnical"), // Character.UnicodeBlock.MISCELLANEOUS_TECHNICAL + /* 2400; 243F; Control Pictures */ + UBInfo(0x2400, 0x243F, "ControlPictures"), // Character.UnicodeBlock.CONTROL_PICTURES + /* 2440; 245F; Optical Character Recognition */ + UBInfo(0x2440, 0x245F, "OpticalCharacterRecognition"), // Character.UnicodeBlock.OPTICAL_CHARACTER_RECOGNITION + /* 2460; 24FF; Enclosed Alphanumerics */ + UBInfo(0x2460, 0x24FF, "EnclosedAlphanumerics"), // Character.UnicodeBlock.ENCLOSED_ALPHANUMERICS + /* 2500; 257F; Box Drawing */ + UBInfo(0x2500, 0x257F, "BoxDrawing"), // Character.UnicodeBlock.BOX_DRAWING + /* 2580; 259F; Block Elements */ + UBInfo(0x2580, 0x259F, "BlockElements"), // Character.UnicodeBlock.BLOCK_ELEMENTS + /* 25A0; 25FF; Geometric Shapes */ + UBInfo(0x25A0, 0x25FF, "GeometricShapes"), // Character.UnicodeBlock.GEOMETRIC_SHAPES + /* 2600; 26FF; Miscellaneous Symbols */ + UBInfo(0x2600, 0x26FF, "MiscellaneousSymbols"), // Character.UnicodeBlock.MISCELLANEOUS_SYMBOLS + /* 2700; 27BF; Dingbats */ + UBInfo(0x2700, 0x27BF, "Dingbats"), // Character.UnicodeBlock.DINGBATS + /* 2800; 28FF; Braille Patterns */ + UBInfo(0x2800, 0x28FF, "BraillePatterns"), // Character.UnicodeBlock.BRAILLE_PATTERNS + /* 2E80; 2EFF; CJK Radicals Supplement */ + UBInfo(0x2E80, 0x2EFF, "CJKRadicalsSupplement"), // Character.UnicodeBlock.CJK_RADICALS_SUPPLEMENT + /* 2F00; 2FDF; Kangxi Radicals */ + UBInfo(0x2F00, 0x2FDF, "KangxiRadicals"), // Character.UnicodeBlock.KANGXI_RADICALS + /* 2FF0; 2FFF; Ideographic Description Characters */ + UBInfo(0x2FF0, 0x2FFF, "IdeographicDescriptionCharacters"), // Character.UnicodeBlock.IDEOGRAPHIC_DESCRIPTION_CHARACTERS + /* 3000; 303F; CJK Symbols and Punctuation */ + UBInfo(0x3000, 0x303F, "CJKSymbolsandPunctuation"), // Character.UnicodeBlock.CJK_SYMBOLS_AND_PUNCTUATION + /* 3040; 309F; Hiragana */ + UBInfo(0x3040, 0x309F, "Hiragana"), // Character.UnicodeBlock.HIRAGANA + /* 30A0; 30FF; Katakana */ + UBInfo(0x30A0, 0x30FF, "Katakana"), // Character.UnicodeBlock.KATAKANA + /* 3100; 312F; Bopomofo */ + UBInfo(0x3100, 0x312F, "Bopomofo"), // Character.UnicodeBlock.BOPOMOFO + /* 3130; 318F; Hangul Compatibility Jamo */ + UBInfo(0x3130, 0x318F, "HangulCompatibilityJamo"), // Character.UnicodeBlock.HANGUL_COMPATIBILITY_JAMO + /* 3190; 319F; Kanbun */ + UBInfo(0x3190, 0x319F, "Kanbun"), // Character.UnicodeBlock.KANBUN + /* 31A0; 31BF; Bopomofo Extended */ + UBInfo(0x31A0, 0x31BF, "BopomofoExtended"), // Character.UnicodeBlock.BOPOMOFO_EXTENDED + /* 3200; 32FF; Enclosed CJK Letters and Months */ + UBInfo(0x3200, 0x32FF, "EnclosedCJKLettersandMonths"), // Character.UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS + /* 3300; 33FF; CJK Compatibility */ + UBInfo(0x3300, 0x33FF, "CJKCompatibility"), // Character.UnicodeBlock.CJK_COMPATIBILITY + /* 3400; 4DB5; CJK Unified Ideographs Extension A */ + UBInfo(0x3400, 0x4DB5, "CJKUnifiedIdeographsExtensionA"), // Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A + /* 4E00; 9FFF; CJK Unified Ideographs */ + UBInfo(0x4E00, 0x9FFF, "CJKUnifiedIdeographs"), // Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS + /* A000; A48F; Yi Syllables */ + UBInfo(0xA000, 0xA48F, "YiSyllables"), // Character.UnicodeBlock.YI_SYLLABLES + /* A490; A4CF; Yi Radicals */ + UBInfo(0xA490, 0xA4CF, "YiRadicals"), // Character.UnicodeBlock.YI_RADICALS + /* AC00; D7A3; Hangul Syllables */ + UBInfo(0xAC00, 0xD7A3, "HangulSyllables"), // Character.UnicodeBlock.HANGUL_SYLLABLES + /* D800; DB7F; High Surrogates */ + /* DB80; DBFF; High Private Use Surrogates */ + /* DC00; DFFF; Low Surrogates */ + /* E000; F8FF; Private Use */ + /* F900; FAFF; CJK Compatibility Ideographs */ + UBInfo(0xF900, 0xFAFF, "CJKCompatibilityIdeographs"), // Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS + /* FB00; FB4F; Alphabetic Presentation Forms */ + UBInfo(0xFB00, 0xFB4F, "AlphabeticPresentationForms"), // Character.UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS + /* FB50; FDFF; Arabic Presentation Forms-A */ + UBInfo(0xFB50, 0xFDFF, "ArabicPresentationForms-A"), // Character.UnicodeBlock.ARABIC_PRESENTATION_FORMS_A + /* FE20; FE2F; Combining Half Marks */ + UBInfo(0xFE20, 0xFE2F, "CombiningHalfMarks"), // Character.UnicodeBlock.COMBINING_HALF_MARKS + /* FE30; FE4F; CJK Compatibility Forms */ + UBInfo(0xFE30, 0xFE4F, "CJKCompatibilityForms"), // Character.UnicodeBlock.CJK_COMPATIBILITY_FORMS + /* FE50; FE6F; Small Form Variants */ + UBInfo(0xFE50, 0xFE6F, "SmallFormVariants"), // Character.UnicodeBlock.SMALL_FORM_VARIANTS + /* FE70; FEFE; Arabic Presentation Forms-B */ + // new UBInfo (0xFE70,0xFEFE,"InArabicPresentationForms-B"), // + // Character.UnicodeBlock.ARABIC_PRESENTATION_FORMS_B + /* FEFF; FEFF; Specials */ + UBInfo(0xFEFF, 0xFEFF, "Specials"), // Character.UnicodeBlock.SPECIALS + /* FF00; FFEF; Halfwidth and Fullwidth Forms */ + UBInfo(0xFF00, 0xFFEF, "HalfwidthandFullwidthForms"), // Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS + /* FFF0; FFFD; Specials */ + UBInfo(0xFFF0, 0xFFFD, "Specials") // Character.UnicodeBlock.SPECIALS +) - // A table representing the unicode character blocks - private val UBlocks = arrayOf( - /* 0000; 007F; Basic Latin */ - UBInfo(0x0000, 0x007F, "BasicLatin"), // Character.UnicodeBlock.BASIC_LATIN - /* 0080; 00FF; Latin-1 Supplement */ - UBInfo(0x0080, 0x00FF, "Latin-1Supplement"), // Character.UnicodeBlock.LATIN_1_SUPPLEMENT - /* 0100; 017F; Latin Extended-A */ - UBInfo(0x0100, 0x017F, "LatinExtended-A"), // Character.UnicodeBlock.LATIN_EXTENDED_A - /* 0180; 024F; Latin Extended-B */ - // new UBInfo (0x0180,0x024F,"InLatinExtended-B"), // - // Character.UnicodeBlock.LATIN_EXTENDED_B - /* 0250; 02AF; IPA Extensions */ - UBInfo(0x0250, 0x02AF, "IPAExtensions"), // Character.UnicodeBlock.IPA_EXTENSIONS - /* 02B0; 02FF; Spacing Modifier Letters */ - UBInfo(0x02B0, 0x02FF, "SpacingModifierLetters"), // Character.UnicodeBlock.SPACING_MODIFIER_LETTERS - /* 0300; 036F; Combining Diacritical Marks */ - UBInfo(0x0300, 0x036F, "CombiningDiacriticalMarks"), // Character.UnicodeBlock.COMBINING_DIACRITICAL_MARKS - /* 0370; 03FF; Greek */ - UBInfo(0x0370, 0x03FF, "Greek"), // Character.UnicodeBlock.GREEK - /* 0400; 04FF; Cyrillic */ - UBInfo(0x0400, 0x04FF, "Cyrillic"), // Character.UnicodeBlock.CYRILLIC - /* 0530; 058F; Armenian */ - UBInfo(0x0530, 0x058F, "Armenian"), // Character.UnicodeBlock.ARMENIAN - /* 0590; 05FF; Hebrew */ - UBInfo(0x0590, 0x05FF, "Hebrew"), // Character.UnicodeBlock.HEBREW - /* 0600; 06FF; Arabic */ - UBInfo(0x0600, 0x06FF, "Arabic"), // Character.UnicodeBlock.ARABIC - /* 0700; 074F; Syriac */ - UBInfo(0x0700, 0x074F, "Syriac"), // Character.UnicodeBlock.SYRIAC - /* 0780; 07BF; Thaana */ - UBInfo(0x0780, 0x07BF, "Thaana"), // Character.UnicodeBlock.THAANA - /* 0900; 097F; Devanagari */ - UBInfo(0x0900, 0x097F, "Devanagari"), // Character.UnicodeBlock.DEVANAGARI - /* 0980; 09FF; Bengali */ - UBInfo(0x0980, 0x09FF, "Bengali"), // Character.UnicodeBlock.BENGALI - /* 0A00; 0A7F; Gurmukhi */ - UBInfo(0x0A00, 0x0A7F, "Gurmukhi"), // Character.UnicodeBlock.GURMUKHI - /* 0A80; 0AFF; Gujarati */ - UBInfo(0x0A80, 0x0AFF, "Gujarati"), // Character.UnicodeBlock.GUJARATI - /* 0B00; 0B7F; Oriya */ - UBInfo(0x0B00, 0x0B7F, "Oriya"), // Character.UnicodeBlock.ORIYA - /* 0B80; 0BFF; Tamil */ - UBInfo(0x0B80, 0x0BFF, "Tamil"), // Character.UnicodeBlock.TAMIL - /* 0C00; 0C7F; Telugu */ - UBInfo(0x0C00, 0x0C7F, "Telugu"), // Character.UnicodeBlock.TELUGU - /* 0C80; 0CFF; Kannada */ - UBInfo(0x0C80, 0x0CFF, "Kannada"), // Character.UnicodeBlock.KANNADA - /* 0D00; 0D7F; Malayalam */ - UBInfo(0x0D00, 0x0D7F, "Malayalam"), // Character.UnicodeBlock.MALAYALAM - /* 0D80; 0DFF; Sinhala */ - UBInfo(0x0D80, 0x0DFF, "Sinhala"), // Character.UnicodeBlock.SINHALA - /* 0E00; 0E7F; Thai */ - UBInfo(0x0E00, 0x0E7F, "Thai"), // Character.UnicodeBlock.THAI - /* 0E80; 0EFF; Lao */ - UBInfo(0x0E80, 0x0EFF, "Lao"), // Character.UnicodeBlock.LAO - /* 0F00; 0FFF; Tibetan */ - UBInfo(0x0F00, 0x0FFF, "Tibetan"), // Character.UnicodeBlock.TIBETAN - /* 1000; 109F; Myanmar */ - UBInfo(0x1000, 0x109F, "Myanmar"), // Character.UnicodeBlock.MYANMAR - /* 10A0; 10FF; Georgian */ - UBInfo(0x10A0, 0x10FF, "Georgian"), // Character.UnicodeBlock.GEORGIAN - /* 1100; 11FF; Hangul Jamo */ - UBInfo(0x1100, 0x11FF, "HangulJamo"), // Character.UnicodeBlock.HANGUL_JAMO - /* 1200; 137F; Ethiopic */ - UBInfo(0x1200, 0x137F, "Ethiopic"), // Character.UnicodeBlock.ETHIOPIC - /* 13A0; 13FF; Cherokee */ - UBInfo(0x13A0, 0x13FF, "Cherokee"), // Character.UnicodeBlock.CHEROKEE - /* 1400; 167F; Unified Canadian Aboriginal Syllabics */ - UBInfo(0x1400, 0x167F, "UnifiedCanadianAboriginalSyllabics"), // Character.UnicodeBlock.UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS - /* 1680; 169F; Ogham */ - UBInfo(0x1680, 0x169F, "Ogham"), // Character.UnicodeBlock.OGHAM - /* 16A0; 16FF; Runic */ - UBInfo(0x16A0, 0x16FF, "Runic"), // Character.UnicodeBlock.RUNIC - /* 1780; 17FF; Khmer */ - UBInfo(0x1780, 0x17FF, "Khmer"), // Character.UnicodeBlock.KHMER - /* 1800; 18AF; Mongolian */ - UBInfo(0x1800, 0x18AF, "Mongolian"), // Character.UnicodeBlock.MONGOLIAN - /* 1E00; 1EFF; Latin Extended Additional */ - UBInfo(0x1E00, 0x1EFF, "LatinExtendedAdditional"), // Character.UnicodeBlock.LATIN_EXTENDED_ADDITIONAL - /* 1F00; 1FFF; Greek Extended */ - UBInfo(0x1F00, 0x1FFF, "GreekExtended"), // Character.UnicodeBlock.GREEK_EXTENDED - /* 2000; 206F; General Punctuation */ - UBInfo(0x2000, 0x206F, "GeneralPunctuation"), // Character.UnicodeBlock.GENERAL_PUNCTUATION - /* 2070; 209F; Superscripts and Subscripts */ - UBInfo(0x2070, 0x209F, "SuperscriptsandSubscripts"), // Character.UnicodeBlock.SUPERSCRIPTS_AND_SUBSCRIPTS - /* 20A0; 20CF; Currency Symbols */ - UBInfo(0x20A0, 0x20CF, "CurrencySymbols"), // Character.UnicodeBlock.CURRENCY_SYMBOLS - /* 20D0; 20FF; Combining Marks for Symbols */ - UBInfo(0x20D0, 0x20FF, "CombiningMarksforSymbols"), // Character.UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS - /* 2100; 214F; Letterlike Symbols */ - UBInfo(0x2100, 0x214F, "LetterlikeSymbols"), // Character.UnicodeBlock.LETTERLIKE_SYMBOLS - /* 2150; 218F; Number Forms */ - UBInfo(0x2150, 0x218F, "NumberForms"), // Character.UnicodeBlock.NUMBER_FORMS - /* 2190; 21FF; Arrows */ - UBInfo(0x2190, 0x21FF, "Arrows"), // Character.UnicodeBlock.ARROWS - /* 2200; 22FF; Mathematical Operators */ - UBInfo(0x2200, 0x22FF, "MathematicalOperators"), // Character.UnicodeBlock.MATHEMATICAL_OPERATORS - /* 2300; 23FF; Miscellaneous Technical */ - UBInfo(0x2300, 0x23FF, "MiscellaneousTechnical"), // Character.UnicodeBlock.MISCELLANEOUS_TECHNICAL - /* 2400; 243F; Control Pictures */ - UBInfo(0x2400, 0x243F, "ControlPictures"), // Character.UnicodeBlock.CONTROL_PICTURES - /* 2440; 245F; Optical Character Recognition */ - UBInfo(0x2440, 0x245F, "OpticalCharacterRecognition"), // Character.UnicodeBlock.OPTICAL_CHARACTER_RECOGNITION - /* 2460; 24FF; Enclosed Alphanumerics */ - UBInfo(0x2460, 0x24FF, "EnclosedAlphanumerics"), // Character.UnicodeBlock.ENCLOSED_ALPHANUMERICS - /* 2500; 257F; Box Drawing */ - UBInfo(0x2500, 0x257F, "BoxDrawing"), // Character.UnicodeBlock.BOX_DRAWING - /* 2580; 259F; Block Elements */ - UBInfo(0x2580, 0x259F, "BlockElements"), // Character.UnicodeBlock.BLOCK_ELEMENTS - /* 25A0; 25FF; Geometric Shapes */ - UBInfo(0x25A0, 0x25FF, "GeometricShapes"), // Character.UnicodeBlock.GEOMETRIC_SHAPES - /* 2600; 26FF; Miscellaneous Symbols */ - UBInfo(0x2600, 0x26FF, "MiscellaneousSymbols"), // Character.UnicodeBlock.MISCELLANEOUS_SYMBOLS - /* 2700; 27BF; Dingbats */ - UBInfo(0x2700, 0x27BF, "Dingbats"), // Character.UnicodeBlock.DINGBATS - /* 2800; 28FF; Braille Patterns */ - UBInfo(0x2800, 0x28FF, "BraillePatterns"), // Character.UnicodeBlock.BRAILLE_PATTERNS - /* 2E80; 2EFF; CJK Radicals Supplement */ - UBInfo(0x2E80, 0x2EFF, "CJKRadicalsSupplement"), // Character.UnicodeBlock.CJK_RADICALS_SUPPLEMENT - /* 2F00; 2FDF; Kangxi Radicals */ - UBInfo(0x2F00, 0x2FDF, "KangxiRadicals"), // Character.UnicodeBlock.KANGXI_RADICALS - /* 2FF0; 2FFF; Ideographic Description Characters */ - UBInfo(0x2FF0, 0x2FFF, "IdeographicDescriptionCharacters"), // Character.UnicodeBlock.IDEOGRAPHIC_DESCRIPTION_CHARACTERS - /* 3000; 303F; CJK Symbols and Punctuation */ - UBInfo(0x3000, 0x303F, "CJKSymbolsandPunctuation"), // Character.UnicodeBlock.CJK_SYMBOLS_AND_PUNCTUATION - /* 3040; 309F; Hiragana */ - UBInfo(0x3040, 0x309F, "Hiragana"), // Character.UnicodeBlock.HIRAGANA - /* 30A0; 30FF; Katakana */ - UBInfo(0x30A0, 0x30FF, "Katakana"), // Character.UnicodeBlock.KATAKANA - /* 3100; 312F; Bopomofo */ - UBInfo(0x3100, 0x312F, "Bopomofo"), // Character.UnicodeBlock.BOPOMOFO - /* 3130; 318F; Hangul Compatibility Jamo */ - UBInfo(0x3130, 0x318F, "HangulCompatibilityJamo"), // Character.UnicodeBlock.HANGUL_COMPATIBILITY_JAMO - /* 3190; 319F; Kanbun */ - UBInfo(0x3190, 0x319F, "Kanbun"), // Character.UnicodeBlock.KANBUN - /* 31A0; 31BF; Bopomofo Extended */ - UBInfo(0x31A0, 0x31BF, "BopomofoExtended"), // Character.UnicodeBlock.BOPOMOFO_EXTENDED - /* 3200; 32FF; Enclosed CJK Letters and Months */ - UBInfo(0x3200, 0x32FF, "EnclosedCJKLettersandMonths"), // Character.UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS - /* 3300; 33FF; CJK Compatibility */ - UBInfo(0x3300, 0x33FF, "CJKCompatibility"), // Character.UnicodeBlock.CJK_COMPATIBILITY - /* 3400; 4DB5; CJK Unified Ideographs Extension A */ - UBInfo(0x3400, 0x4DB5, "CJKUnifiedIdeographsExtensionA"), // Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A - /* 4E00; 9FFF; CJK Unified Ideographs */ - UBInfo(0x4E00, 0x9FFF, "CJKUnifiedIdeographs"), // Character.UnicodeBlock.CJK_UNIFIED_IDEOGRAPHS - /* A000; A48F; Yi Syllables */ - UBInfo(0xA000, 0xA48F, "YiSyllables"), // Character.UnicodeBlock.YI_SYLLABLES - /* A490; A4CF; Yi Radicals */ - UBInfo(0xA490, 0xA4CF, "YiRadicals"), // Character.UnicodeBlock.YI_RADICALS - /* AC00; D7A3; Hangul Syllables */ - UBInfo(0xAC00, 0xD7A3, "HangulSyllables"), // Character.UnicodeBlock.HANGUL_SYLLABLES - /* D800; DB7F; High Surrogates */ - /* DB80; DBFF; High Private Use Surrogates */ - /* DC00; DFFF; Low Surrogates */ - /* E000; F8FF; Private Use */ - /* F900; FAFF; CJK Compatibility Ideographs */ - UBInfo(0xF900, 0xFAFF, "CJKCompatibilityIdeographs"), // Character.UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS - /* FB00; FB4F; Alphabetic Presentation Forms */ - UBInfo(0xFB00, 0xFB4F, "AlphabeticPresentationForms"), // Character.UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS - /* FB50; FDFF; Arabic Presentation Forms-A */ - UBInfo(0xFB50, 0xFDFF, "ArabicPresentationForms-A"), // Character.UnicodeBlock.ARABIC_PRESENTATION_FORMS_A - /* FE20; FE2F; Combining Half Marks */ - UBInfo(0xFE20, 0xFE2F, "CombiningHalfMarks"), // Character.UnicodeBlock.COMBINING_HALF_MARKS - /* FE30; FE4F; CJK Compatibility Forms */ - UBInfo(0xFE30, 0xFE4F, "CJKCompatibilityForms"), // Character.UnicodeBlock.CJK_COMPATIBILITY_FORMS - /* FE50; FE6F; Small Form Variants */ - UBInfo(0xFE50, 0xFE6F, "SmallFormVariants"), // Character.UnicodeBlock.SMALL_FORM_VARIANTS - /* FE70; FEFE; Arabic Presentation Forms-B */ - // new UBInfo (0xFE70,0xFEFE,"InArabicPresentationForms-B"), // - // Character.UnicodeBlock.ARABIC_PRESENTATION_FORMS_B - /* FEFF; FEFF; Specials */ - UBInfo(0xFEFF, 0xFEFF, "Specials"), // Character.UnicodeBlock.SPECIALS - /* FF00; FFEF; Halfwidth and Fullwidth Forms */ - UBInfo(0xFF00, 0xFFEF, "HalfwidthandFullwidthForms"), // Character.UnicodeBlock.HALFWIDTH_AND_FULLWIDTH_FORMS - /* FFF0; FFFD; Specials */ - UBInfo(0xFFF0, 0xFFFD, "Specials") // Character.UnicodeBlock.SPECIALS - ) -} fun box() { testSimpleMatch() diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/ReplaceTest.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/ReplaceTest.kt index 0de953a396b..02dd59872cd 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/ReplaceTest.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/ReplaceTest.kt @@ -28,8 +28,8 @@ fun testSimpleReplace() { val regex = Regex(pattern) - Assert.assertEquals("foobarxxxarfoofo1", regex.replaceFirst(target, repl)) - Assert.assertEquals("foobarxxxarfooxxx", regex.replace(target, repl)) + assertEquals("foobarxxxarfoofo1", regex.replaceFirst(target, repl)) + assertEquals("foobarxxxarfooxxx", regex.replace(target, repl)) } fun testCaptureReplace() { @@ -45,18 +45,18 @@ fun testCaptureReplace() { regex = Regex(pattern) s = regex.replaceFirst(target, repl) - Assert.assertEquals("foo[31];bar[42];[99]xyz", s) + assertEquals("foo[31];bar[42];[99]xyz", s) s = regex.replace(target, repl) - Assert.assertEquals("foo[31];bar[42];xyz[99]", s) + assertEquals("foo[31];bar[42];xyz[99]", s) target = "[31]foo(42)bar{63}zoo;[12]abc(34)def{56}ghi;{99}xyz[88]xyz(77)xyz;" pattern = "\\[([0-9]+)\\]([a-z]+)\\(([0-9]+)\\)([a-z]+)\\{([0-9]+)\\}([a-z]+)" repl = "[$5]$6($3)$4{$1}$2" regex = Regex(pattern) s = regex.replaceFirst(target, repl) - Assert.assertEquals("[63]zoo(42)bar{31}foo;[12]abc(34)def{56}ghi;{99}xyz[88]xyz(77)xyz;", s) + assertEquals("[63]zoo(42)bar{31}foo;[12]abc(34)def{56}ghi;{99}xyz[88]xyz(77)xyz;", s) s = regex.replace(target, repl) - Assert.assertEquals("[63]zoo(42)bar{31}foo;[56]ghi(34)def{12}abc;{99}xyz[88]xyz(77)xyz;", s) + assertEquals("[63]zoo(42)bar{31}foo;[56]ghi(34)def{12}abc;{99}xyz[88]xyz(77)xyz;", s) } fun testEscapeReplace() { @@ -69,13 +69,13 @@ fun testEscapeReplace() { pattern = "'" repl = "\\'" s = target.replace(pattern.toRegex(), repl) - Assert.assertEquals("foo'bar''foo", s) + assertEquals("foo'bar''foo", s) repl = "\\\\'" s = target.replace(pattern.toRegex(), repl) - Assert.assertEquals("foo\\'bar\\'\\'foo", s) + assertEquals("foo\\'bar\\'\\'foo", s) repl = "\\$3" s = target.replace(pattern.toRegex(), repl) - Assert.assertEquals("foo$3bar$3$3foo", s) + assertEquals("foo$3bar$3$3foo", s) } fun box() { diff --git a/backend.native/tests/external/stdlib/text/RegexTest/harmony/SplitTest.kt b/backend.native/tests/external/stdlib/text/RegexTest/harmony/SplitTest.kt index 8b7573d7e1e..c977407bf56 100644 --- a/backend.native/tests/external/stdlib/text/RegexTest/harmony/SplitTest.kt +++ b/backend.native/tests/external/stdlib/text/RegexTest/harmony/SplitTest.kt @@ -22,13 +22,12 @@ fun testSimple() { val p = Regex("/") val results = p.split("have/you/done/it/right") val expected = arrayOf("have", "you", "done", "it", "right") - Assert.assertEquals(expected.size, results.size) + assertEquals(expected.size, results.size) for (i in expected.indices) { - Assert.assertEquals(results[i], expected[i]) + assertEquals(results[i], expected[i]) } } -@Throws(PatternSyntaxException::class) fun testSplit1() { var p = Regex(" ") @@ -36,100 +35,100 @@ fun testSplit1() { var tokens: List tokens = p.split(input, 1) - Assert.assertEquals(1, tokens.size) - Assert.assertTrue(tokens[0] == input) + assertEquals(1, tokens.size) + assertTrue(tokens[0] == input) tokens = p.split(input, 2) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poodle", tokens[0]) - Assert.assertEquals("zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poodle", tokens[0]) + assertEquals("zoo", tokens[1]) tokens = p.split(input, 5) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poodle", tokens[0]) - Assert.assertEquals("zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poodle", tokens[0]) + assertEquals("zoo", tokens[1]) tokens = p.split(input, 0) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poodle", tokens[0]) - Assert.assertEquals("zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poodle", tokens[0]) + assertEquals("zoo", tokens[1]) tokens = p.split(input) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poodle", tokens[0]) - Assert.assertEquals("zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poodle", tokens[0]) + assertEquals("zoo", tokens[1]) p = Regex("d") tokens = p.split(input, 1) - Assert.assertEquals(1, tokens.size) - Assert.assertTrue(tokens[0] == input) + assertEquals(1, tokens.size) + assertTrue(tokens[0] == input) tokens = p.split(input, 2) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poo", tokens[0]) - Assert.assertEquals("le zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poo", tokens[0]) + assertEquals("le zoo", tokens[1]) tokens = p.split(input, 5) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poo", tokens[0]) - Assert.assertEquals("le zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poo", tokens[0]) + assertEquals("le zoo", tokens[1]) tokens = p.split(input, 0) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poo", tokens[0]) - Assert.assertEquals("le zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poo", tokens[0]) + assertEquals("le zoo", tokens[1]) tokens = p.split(input) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("poo", tokens[0]) - Assert.assertEquals("le zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("poo", tokens[0]) + assertEquals("le zoo", tokens[1]) p = Regex("o") tokens = p.split(input, 1) - Assert.assertEquals(1, tokens.size) - Assert.assertTrue(tokens[0] == input) + assertEquals(1, tokens.size) + assertTrue(tokens[0] == input) tokens = p.split(input, 2) - Assert.assertEquals(2, tokens.size) - Assert.assertEquals("p", tokens[0]) - Assert.assertEquals("odle zoo", tokens[1]) + assertEquals(2, tokens.size) + assertEquals("p", tokens[0]) + assertEquals("odle zoo", tokens[1]) tokens = p.split(input, 5) - Assert.assertEquals(5, tokens.size) - Assert.assertEquals("p", tokens[0]) - Assert.assertTrue(tokens[1] == "") - Assert.assertEquals("dle z", tokens[2]) - Assert.assertTrue(tokens[3] == "") - Assert.assertTrue(tokens[4] == "") + assertEquals(5, tokens.size) + assertEquals("p", tokens[0]) + assertTrue(tokens[1] == "") + assertEquals("dle z", tokens[2]) + assertTrue(tokens[3] == "") + assertTrue(tokens[4] == "") tokens = p.split(input, 0) - Assert.assertEquals(5, tokens.size) - Assert.assertEquals("p", tokens[0]) - Assert.assertTrue(tokens[1] == "") - Assert.assertEquals("dle z", tokens[2]) - Assert.assertTrue(tokens[3] == "") - Assert.assertTrue(tokens[4] == "") + assertEquals(5, tokens.size) + assertEquals("p", tokens[0]) + assertTrue(tokens[1] == "") + assertEquals("dle z", tokens[2]) + assertTrue(tokens[3] == "") + assertTrue(tokens[4] == "") tokens = p.split(input) - Assert.assertEquals(5, tokens.size) - Assert.assertEquals("p", tokens[0]) - Assert.assertTrue(tokens[1] == "") - Assert.assertEquals("dle z", tokens[2]) - Assert.assertTrue(tokens[3] == "") - Assert.assertTrue(tokens[4] == "") + assertEquals(5, tokens.size) + assertEquals("p", tokens[0]) + assertTrue(tokens[1] == "") + assertEquals("dle z", tokens[2]) + assertTrue(tokens[3] == "") + assertTrue(tokens[4] == "") } fun testSplit2() { val p = Regex("") var s: List s = p.split("a", 0) - Assert.assertEquals(3, s.size) - Assert.assertEquals("", s[0]) - Assert.assertEquals("a", s[1]) - Assert.assertEquals("", s[2]) + assertEquals(3, s.size) + assertEquals("", s[0]) + assertEquals("a", s[1]) + assertEquals("", s[2]) s = p.split("", 0) - Assert.assertEquals(1, s.size) - Assert.assertEquals("", s[0]) + assertEquals(1, s.size) + assertEquals("", s[0]) s = p.split("abcd", 0) - Assert.assertEquals(6, s.size) - Assert.assertEquals("", s[0]) - Assert.assertEquals("a", s[1]) - Assert.assertEquals("b", s[2]) - Assert.assertEquals("c", s[3]) - Assert.assertEquals("d", s[4]) - Assert.assertEquals("", s[5]) + assertEquals(6, s.size) + assertEquals("", s[0]) + assertEquals("a", s[1]) + assertEquals("b", s[2]) + assertEquals("c", s[3]) + assertEquals("d", s[4]) + assertEquals("", s[5]) } fun testSplitSupplementaryWithEmptyString() { @@ -141,12 +140,12 @@ fun testSplitSupplementaryWithEmptyString() { val p = Regex("") val s: List s = p.split("a\ud869\uded6b", 0) - Assert.assertEquals(5, s.size) - Assert.assertEquals("", s[0]) - Assert.assertEquals("a", s[1]) - Assert.assertEquals("\ud869\uded6", s[2]) - Assert.assertEquals("b", s[3]) - Assert.assertEquals("", s[4]) + assertEquals(5, s.size) + assertEquals("", s[0]) + assertEquals("a", s[1]) + assertEquals("\ud869\uded6", s[2]) + assertEquals("b", s[3]) + assertEquals("", s[4]) } fun box() { diff --git a/runtime/src/main/kotlin/kotlin/text/Strings.kt b/runtime/src/main/kotlin/kotlin/text/Strings.kt index 5945810515e..516e01c00ca 100644 --- a/runtime/src/main/kotlin/kotlin/text/Strings.kt +++ b/runtime/src/main/kotlin/kotlin/text/Strings.kt @@ -647,35 +647,31 @@ public fun String.replaceBeforeLast(delimiter: String, replacement: String, miss * The [replacement] can consist of any combination of literal text and $-substitutions. To treat the replacement string * literally escape it with the [kotlin.text.Regex.Companion.escapeReplacement] method. */ -//@FixmeRegex -//@kotlin.internal.InlineOnly -//public inline fun CharSequence.replace(regex: Regex, replacement: String): String = regex.replace(this, replacement) +@kotlin.internal.InlineOnly +public inline fun CharSequence.replace(regex: Regex, replacement: String): String = regex.replace(this, replacement) /** * Returns a new string obtained by replacing each substring of this char sequence that matches the given regular expression * with the result of the given function [transform] that takes [MatchResult] and returns a string to be used as a * replacement for that match. */ -//@FixmeRegex -//@kotlin.internal.InlineOnly -//public inline fun CharSequence.replace(regex: Regex, noinline transform: (MatchResult) -> CharSequence): String = regex.replace(this, transform) +@kotlin.internal.InlineOnly +public inline fun CharSequence.replace(regex: Regex, noinline transform: (MatchResult) -> CharSequence): String = regex.replace(this, transform) /** * Replaces the first occurrence of the given regular expression [regex] in this char sequence with specified [replacement] expression. * * @param replacement A replacement expression that can include substitutions. See [Regex.replaceFirst] for details. */ -//@FixmeRegex -//@kotlin.internal.InlineOnly -//public inline fun CharSequence.replaceFirst(regex: Regex, replacement: String): String = regex.replaceFirst(this, replacement) +@kotlin.internal.InlineOnly +public inline fun CharSequence.replaceFirst(regex: Regex, replacement: String): String = regex.replaceFirst(this, replacement) /** * Returns `true` if this char sequence matches the given regular expression. */ -//@FixmeRegex -//@kotlin.internal.InlineOnly -//public inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this) +@kotlin.internal.InlineOnly +public inline fun CharSequence.matches(regex: Regex): Boolean = regex.matches(this) /** * Implementation of [regionMatches] for CharSequences. @@ -1017,9 +1013,8 @@ public operator fun CharSequence.contains(char: Char, ignoreCase: Boolean = fals /** * Returns `true` if this char sequence contains at least one match of the specified regular expression [regex]. */ -//@FixmeRegex -//@kotlin.internal.InlineOnly -//public inline operator fun CharSequence.contains(regex: Regex): Boolean = regex.containsMatchIn(this) +@kotlin.internal.InlineOnly +public inline operator fun CharSequence.contains(regex: Regex): Boolean = regex.containsMatchIn(this) // rangesDelimitedBy @@ -1177,9 +1172,8 @@ public fun CharSequence.split(vararg delimiters: Char, ignoreCase: Boolean = fal * @param limit Non-negative value specifying the maximum number of substrings to return. * Zero by default means no limit is set. */ -//@FixmeRegex -//@kotlin.internal.InlineOnly -//public inline fun CharSequence.split(regex: Regex, limit: Int = 0): List = regex.split(this, limit) +@kotlin.internal.InlineOnly +public inline fun CharSequence.split(regex: Regex, limit: Int = 0): List = regex.split(this, limit) /** * Splits this char sequence to a sequence of lines delimited by any of the following character sequences: CRLF, LF or CR. @@ -2391,3 +2385,22 @@ public fun CharSequence.repeat(n: Int): String { } } } + + +/** + * Converts the string into a regular expression [Regex] with the default options. + */ +@kotlin.internal.InlineOnly +public inline fun String.toRegex(): Regex = Regex(this) + +/** + * Converts the string into a regular expression [Regex] with the specified single [option]. + */ +@kotlin.internal.InlineOnly +public inline fun String.toRegex(option: RegexOption): Regex = Regex(this, option) + +/** + * Converts the string into a regular expression [Regex] with the specified set of [options]. + */ +@kotlin.internal.InlineOnly +public inline fun String.toRegex(options: Set): Regex = Regex(this, options) diff --git a/runtime/src/main/kotlin/kotlin/text/regex/AbstractCharClass.kt b/runtime/src/main/kotlin/kotlin/text/regex/AbstractCharClass.kt new file mode 100644 index 00000000000..ad5455db132 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/AbstractCharClass.kt @@ -0,0 +1,553 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Unicode category (i.e. Ll, Lu). + * @author Nikolay A. Kuznetsov + */ +internal open class UnicodeCategory(protected val category: Int) : AbstractCharClass() { + override fun contains(ch: Int): Boolean = alt xor (ch.toChar().category.value == category) +} + +/** + * Unicode category scope (i.e IsL, IsM, ...) + * @author Nikolay A. Kuznetsov + */ +internal class UnicodeCategoryScope(category: Int) : UnicodeCategory(category) { + override fun contains(ch: Int): Boolean = alt xor (category shr ch.toChar().category.value and 1 != 0) +} + +/** + * This class represents character classes, i.e. sets of character either predefined or user defined. + * Note: this class represent a token, not node, so being constructed by lexer. + * + * @author Nikolay A. Kuznetsov + */ +internal abstract class AbstractCharClass : SpecialToken() { + + /** + * Show if the class has alternative meaning: + * if the class contains character 'a' and alt == true then the class will contains all characters except 'a'. + */ + internal var alt: Boolean = false + internal var altSurrogates: Boolean = false + + internal val lowHighSurrogates = BitSet(SURROGATE_CARDINALITY) // Bit set for surrogates? + + /* + * Indicates if this class may contain supplementary Unicode codepoints. + * If this flag is specified it doesn't mean that this class contains supplementary characters but may contain. + */ + var mayContainSupplCodepoints = false + protected set + + /** Returns true if this char class contains character specified. */ + abstract operator fun contains(ch: Int): Boolean + open fun contains(ch: Char): Boolean = contains(ch.toInt()) + + /** + * Returns BitSet representing this character class or `null` + * if this character class does not have character representation; + */ + // TODO: @C++?. Or implement a BitSet + open internal val bits: BitSet? + get() = null + + fun hasLowHighSurrogates(): Boolean { + return if (altSurrogates) + lowHighSurrogates.nextClearBit(0) != -1 // TODO: What if the bitset is empty? + else + lowHighSurrogates.nextSetBit(0) != -1 + } + + override val type: Type = Type.CHARCLASS + + open val instance: AbstractCharClass + get() = this + + // TODO: refactor + val surrogates: AbstractCharClass by lazy { + val result = object : AbstractCharClass() { + override fun contains(ch: Int): Boolean { + val index = ch - Char.MIN_SURROGATE.toInt() + + return if (index >= 0 && index < AbstractCharClass.SURROGATE_CARDINALITY) + this.altSurrogates xor this@AbstractCharClass.lowHighSurrogates.get(index) + else + false + } + } + result.setNegative(this.altSurrogates) + return@lazy result + } + + + // TODO: refactor + val withoutSurrogates: AbstractCharClass by lazy { + val result = object : AbstractCharClass() { + override fun contains(ch: Int): Boolean { + val index = ch - Char.MIN_SURROGATE.toInt() + + val containslHS = if (index >= 0 && index < AbstractCharClass.SURROGATE_CARDINALITY) + this.altSurrogates xor this@AbstractCharClass.lowHighSurrogates.get(index) + else + false + + + return this@AbstractCharClass.contains(ch) && !containslHS + } + } + result.setNegative(isNegative()) + result.mayContainSupplCodepoints = mayContainSupplCodepoints + return@lazy result + } + + /** + * Sets this CharClass to negative form, i.e. if they will add some characters and after that set this + * class to negative it will accept all the characters except previously set ones. + * + * Although this method will not alternate all the already set characters, + * just overall meaning of the class. + */ + // TODO: replace with a property + fun setNegative(value: Boolean): AbstractCharClass { + if (alt xor value) { + alt = !alt + altSurrogates = !altSurrogates + } + if (!mayContainSupplCodepoints) { + mayContainSupplCodepoints = true + } + return this + } + + fun isNegative(): Boolean { + return alt + } + + // TODO: replace getValue with property access? + internal abstract class LazyCharClass { + private val posValue: AbstractCharClass by lazy { computeValue() } + private val negValue: AbstractCharClass by lazy { computeValue().setNegative(true) } + + fun getValue(negative: Boolean): AbstractCharClass = if (!negative) posValue else negValue + protected abstract fun computeValue(): AbstractCharClass + } + + internal class LazyDigit : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add('0', '9') + } + } + + // TODO: DO we need it? But we have LazyCharCLass.negValue. May be it's connected with mayContainSupplCodepoints? + // TODO: Don't use bitmaps in preset classes? + internal class LazyNonDigit : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + val result = CharClass().add('0', '9').setNegative(true) + + result.mayContainSupplCodepoints = true + return result + } + } + + internal class LazySpace : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + /* 9-13 - \t\n\x0B\f\r; 32 - ' ' */ + return CharClass().add(9, 13).add(32) + } + } + + internal class LazyNonSpace : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + val result = LazySpace().getValue(negative = true) + + result.mayContainSupplCodepoints = true + return result + } + } + + internal class LazyWord : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add('a', 'z').add('A', 'Z').add('0', '9') + .add('_') + } + } + + internal class LazyNonWord : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + val result = LazyWord().getValue(negative = true) + + result.mayContainSupplCodepoints = true + return result + } + } + + // TODO: It is only for latin in Harmony. + internal class LazyLower : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add('a', 'z') + } + } + + internal class LazyUpper : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add('A', 'Z') + } + } + + internal class LazyASCII : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add(0x00, 0x7F) + } + } + + internal class LazyAlpha : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add('a', 'z').add('A', 'Z') + } + } + + internal class LazyAlnum : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + // TODO: Get rid of the cast? + return (LazyAlpha().getValue(negative = false) as CharClass).add('0', '9') + } + } + + internal class LazyPunct : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + /* Punctuation !"#$%&'()*+,-./:;<=>?@ [\]^_` {|}~ */ + return CharClass().add(0x21, 0x40).add(0x5B, 0x60).add(0x7B, + 0x7E) + } + } + + internal class LazyGraph : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + /* plus punctuation */ + return (LazyAlnum().getValue(negative = false) as CharClass) + .add(0x21, 0x40) + .add(0x5B, 0x60) + .add(0x7B, 0x7E) + } + } + + internal class LazyPrint : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return (LazyGraph().getValue(negative = true) as CharClass).add(0x20) + } + } + + internal class LazyBlank : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add(' ').add('\t') + } + } + + internal class LazyCntrl : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add(0x00, 0x1F).add(0x7F) + } + } + + internal class LazyXDigit : LazyCharClass() { + override fun computeValue(): AbstractCharClass { + return CharClass().add('0', '9').add('a', 'f').add('A', 'F') + } + } + + internal class LazyRange(var start: Int, var end: Int) : LazyCharClass() { + + public override fun computeValue(): AbstractCharClass { + val chCl = CharClass().add(start, end) + return chCl + } + } + + internal class LazySpecialsBlock : LazyCharClass() { + public override fun computeValue(): AbstractCharClass { + return CharClass().add(0xFEFF, 0xFEFF).add(0xFFF0, 0xFFFD) + } + } + + internal class LazyCategoryScope( + val category: Int, + val mayContainSupplCodepoints: Boolean, + val containsAllSurrogates: Boolean = false) : LazyCharClass() { + + override fun computeValue(): AbstractCharClass { + val result = UnicodeCategoryScope(category) + if (containsAllSurrogates) { + result.lowHighSurrogates.set(0, SURROGATE_CARDINALITY) + } + + result.mayContainSupplCodepoints = mayContainSupplCodepoints + return result + } + } + + internal class LazyCategory( + val category: Int, + val mayContainSupplCodepoints: Boolean, + val containsAllSurrogates: Boolean = false) : LazyCharClass() { + + override fun computeValue(): AbstractCharClass { + val result = UnicodeCategory(category) + if (containsAllSurrogates) { + result.lowHighSurrogates.set(0, SURROGATE_CARDINALITY) + } + result.mayContainSupplCodepoints = mayContainSupplCodepoints + return result + } + } + + // ----------------------------------------------------------------- + // Static methods and predefined classes + // ----------------------------------------------------------------- + companion object { + + //Char.MAX_SURROGATE - Char.MIN_SURROGATE + 1 + var SURROGATE_CARDINALITY = 2048 + + var space: LazyCharClass = LazySpace() + var digit: LazyCharClass = LazyDigit() + + /** + * character classes generated from + * http://www.unicode.org/reports/tr18/ + * http://www.unicode.org/Public/4.1.0/ucd/Blocks.txt + */ + // TODO: @C++ Harmony uses ListResourceBundle class here. TODO: perfrom a effector lookup on the C++ side. + // Temporary solution: when + // TODO: @C++. Or make it in code. Or do something else. + // We can lazily create objects of char classes in C++ hashmap or Array. + // The main point of it is laziness + fun getClass(name: String) : LazyCharClass { + + return when (name) { + + "Lower" -> LazyLower() + "Upper" -> LazyUpper() + "ASCII" -> LazyASCII() + "Alpha" -> LazyAlpha() + "Digit" -> digit + "Alnum" -> LazyAlnum() + "Punct" -> LazyPunct() + "Graph" -> LazyGraph() + "Print" -> LazyPrint() + "Blank" -> LazyBlank() + "Cntrl" -> LazyCntrl() + "XDigit" -> LazyXDigit() + "Space" -> space + "w" -> LazyWord() + "W" -> LazyNonWord() + "s" -> space + "S" -> LazyNonSpace() + "d" -> digit + "D" -> LazyNonDigit() + "BasicLatin" -> LazyRange(0x0000, 0x007F) + "Latin-1Supplement" -> LazyRange(0x0080, 0x00FF) + "LatinExtended-A" -> LazyRange(0x0100, 0x017F) + "LatinExtended-B" -> LazyRange(0x0180, 0x024F) + "IPAExtensions" -> LazyRange(0x0250, 0x02AF) + "SpacingModifierLetters" -> LazyRange(0x02B0, 0x02FF) + "CombiningDiacriticalMarks" -> LazyRange(0x0300, 0x036F) + "Greek" -> LazyRange(0x0370, 0x03FF) + "Cyrillic" -> LazyRange(0x0400, 0x04FF) + "CyrillicSupplement" -> LazyRange(0x0500, 0x052F) + "Armenian" -> LazyRange(0x0530, 0x058F) + "Hebrew" -> LazyRange(0x0590, 0x05FF) + "Arabic" -> LazyRange(0x0600, 0x06FF) + "Syriac" -> LazyRange(0x0700, 0x074F) + "ArabicSupplement" -> LazyRange(0x0750, 0x077F) + "Thaana" -> LazyRange(0x0780, 0x07BF) + "Devanagari" -> LazyRange(0x0900, 0x097F) + "Bengali" -> LazyRange(0x0980, 0x09FF) + "Gurmukhi" -> LazyRange(0x0A00, 0x0A7F) + "Gujarati" -> LazyRange(0x0A80, 0x0AFF) + "Oriya" -> LazyRange(0x0B00, 0x0B7F) + "Tamil" -> LazyRange(0x0B80, 0x0BFF) + "Telugu" -> LazyRange(0x0C00, 0x0C7F) + "Kannada" -> LazyRange(0x0C80, 0x0CFF) + "Malayalam" -> LazyRange(0x0D00, 0x0D7F) + "Sinhala" -> LazyRange(0x0D80, 0x0DFF) + "Thai" -> LazyRange(0x0E00, 0x0E7F) + "Lao" -> LazyRange(0x0E80, 0x0EFF) + "Tibetan" -> LazyRange(0x0F00, 0x0FFF) + "Myanmar" -> LazyRange(0x1000, 0x109F) + "Georgian" -> LazyRange(0x10A0, 0x10FF) + "HangulJamo" -> LazyRange(0x1100, 0x11FF) + "Ethiopic" -> LazyRange(0x1200, 0x137F) + "EthiopicSupplement" -> LazyRange(0x1380, 0x139F) + "Cherokee" -> LazyRange(0x13A0, 0x13FF) + "UnifiedCanadianAboriginalSyllabics" -> LazyRange(0x1400, 0x167F) + "Ogham" -> LazyRange(0x1680, 0x169F) + "Runic" -> LazyRange(0x16A0, 0x16FF) + "Tagalog" -> LazyRange(0x1700, 0x171F) + "Hanunoo" -> LazyRange(0x1720, 0x173F) + "Buhid" -> LazyRange(0x1740, 0x175F) + "Tagbanwa" -> LazyRange(0x1760, 0x177F) + "Khmer" -> LazyRange(0x1780, 0x17FF) + "Mongolian" -> LazyRange(0x1800, 0x18AF) + "Limbu" -> LazyRange(0x1900, 0x194F) + "TaiLe" -> LazyRange(0x1950, 0x197F) + "NewTaiLue" -> LazyRange(0x1980, 0x19DF) + "KhmerSymbols" -> LazyRange(0x19E0, 0x19FF) + "Buginese" -> LazyRange(0x1A00, 0x1A1F) + "PhoneticExtensions" -> LazyRange(0x1D00, 0x1D7F) + "PhoneticExtensionsSupplement" -> LazyRange(0x1D80, 0x1DBF) + "CombiningDiacriticalMarksSupplement" -> LazyRange(0x1DC0, 0x1DFF) + "LatinExtendedAdditional" -> LazyRange(0x1E00, 0x1EFF) + "GreekExtended" -> LazyRange(0x1F00, 0x1FFF) + "GeneralPunctuation" -> LazyRange(0x2000, 0x206F) + "SuperscriptsandSubscripts" -> LazyRange(0x2070, 0x209F) + "CurrencySymbols" -> LazyRange(0x20A0, 0x20CF) + "CombiningMarksforSymbols" -> LazyRange(0x20D0, 0x20FF) + "LetterlikeSymbols" -> LazyRange(0x2100, 0x214F) + "NumberForms" -> LazyRange(0x2150, 0x218F) + "Arrows" -> LazyRange(0x2190, 0x21FF) + "MathematicalOperators" -> LazyRange(0x2200, 0x22FF) + "MiscellaneousTechnical" -> LazyRange(0x2300, 0x23FF) + "ControlPictures" -> LazyRange(0x2400, 0x243F) + "OpticalCharacterRecognition" -> LazyRange(0x2440, 0x245F) + "EnclosedAlphanumerics" -> LazyRange(0x2460, 0x24FF) + "BoxDrawing" -> LazyRange(0x2500, 0x257F) + "BlockElements" -> LazyRange(0x2580, 0x259F) + "GeometricShapes" -> LazyRange(0x25A0, 0x25FF) + "MiscellaneousSymbols" -> LazyRange(0x2600, 0x26FF) + "Dingbats" -> LazyRange(0x2700, 0x27BF) + "MiscellaneousMathematicalSymbols-A" -> LazyRange(0x27C0, 0x27EF) + "SupplementalArrows-A" -> LazyRange(0x27F0, 0x27FF) + "BraillePatterns" -> LazyRange(0x2800, 0x28FF) + "SupplementalArrows-B" -> LazyRange(0x2900, 0x297F) + "MiscellaneousMathematicalSymbols-B" -> LazyRange(0x2980, 0x29FF) + "SupplementalMathematicalOperators" -> LazyRange(0x2A00, 0x2AFF) + "MiscellaneousSymbolsandArrows" -> LazyRange(0x2B00, 0x2BFF) + "Glagolitic" -> LazyRange(0x2C00, 0x2C5F) + "Coptic" -> LazyRange(0x2C80, 0x2CFF) + "GeorgianSupplement" -> LazyRange(0x2D00, 0x2D2F) + "Tifinagh" -> LazyRange(0x2D30, 0x2D7F) + "EthiopicExtended" -> LazyRange(0x2D80, 0x2DDF) + "SupplementalPunctuation" -> LazyRange(0x2E00, 0x2E7F) + "CJKRadicalsSupplement" -> LazyRange(0x2E80, 0x2EFF) + "KangxiRadicals" -> LazyRange(0x2F00, 0x2FDF) + "IdeographicDescriptionCharacters" -> LazyRange(0x2FF0, 0x2FFF) + "CJKSymbolsandPunctuation" -> LazyRange(0x3000, 0x303F) + "Hiragana" -> LazyRange(0x3040, 0x309F) + "Katakana" -> LazyRange(0x30A0, 0x30FF) + "Bopomofo" -> LazyRange(0x3100, 0x312F) + "HangulCompatibilityJamo" -> LazyRange(0x3130, 0x318F) + "Kanbun" -> LazyRange(0x3190, 0x319F) + "BopomofoExtended" -> LazyRange(0x31A0, 0x31BF) + "CJKStrokes" -> LazyRange(0x31C0, 0x31EF) + "KatakanaPhoneticExtensions" -> LazyRange(0x31F0, 0x31FF) + "EnclosedCJKLettersandMonths" -> LazyRange(0x3200, 0x32FF) + "CJKCompatibility" -> LazyRange(0x3300, 0x33FF) + "CJKUnifiedIdeographsExtensionA" -> LazyRange(0x3400, 0x4DB5) + "YijingHexagramSymbols" -> LazyRange(0x4DC0, 0x4DFF) + "CJKUnifiedIdeographs" -> LazyRange(0x4E00, 0x9FFF) + "YiSyllables" -> LazyRange(0xA000, 0xA48F) + "YiRadicals" -> LazyRange(0xA490, 0xA4CF) + "ModifierToneLetters" -> LazyRange(0xA700, 0xA71F) + "SylotiNagri" -> LazyRange(0xA800, 0xA82F) + "HangulSyllables" -> LazyRange(0xAC00, 0xD7A3) + "HighSurrogates" -> LazyRange(0xD800, 0xDB7F) + "HighPrivateUseSurrogates" -> LazyRange(0xDB80, 0xDBFF) + "LowSurrogates" -> LazyRange(0xDC00, 0xDFFF) + "PrivateUseArea" -> LazyRange(0xE000, 0xF8FF) + "CJKCompatibilityIdeographs" -> LazyRange(0xF900, 0xFAFF) + "AlphabeticPresentationForms" -> LazyRange(0xFB00, 0xFB4F) + "ArabicPresentationForms-A" -> LazyRange(0xFB50, 0xFDFF) + "VariationSelectors" -> LazyRange(0xFE00, 0xFE0F) + "VerticalForms" -> LazyRange(0xFE10, 0xFE1F) + "CombiningHalfMarks" -> LazyRange(0xFE20, 0xFE2F) + "CJKCompatibilityForms" -> LazyRange(0xFE30, 0xFE4F) + "SmallFormVariants" -> LazyRange(0xFE50, 0xFE6F) + "ArabicPresentationForms-B" -> LazyRange(0xFE70, 0xFEFF) + "HalfwidthandFullwidthForms" -> LazyRange(0xFF00, 0xFFEF) + "all" -> LazyRange(0x00, 0x10FFFF) + "Specials" -> LazySpecialsBlock() + "Cn" -> LazyCategory(CharCategory.UNASSIGNED.value, true) + "IsL" -> LazyCategoryScope(0x3E, true) + "Lu" -> LazyCategory(CharCategory.UPPERCASE_LETTER.value, true) + "Ll" -> LazyCategory(CharCategory.LOWERCASE_LETTER.value, true) + "Lt" -> LazyCategory(CharCategory.TITLECASE_LETTER.value, false) + "Lm" -> LazyCategory(CharCategory.MODIFIER_LETTER.value, false) + "Lo" -> LazyCategory(CharCategory.OTHER_LETTER.value, true) + "IsM" -> LazyCategoryScope(0x1C0, true) + "Mn" -> LazyCategory(CharCategory.NON_SPACING_MARK.value, true) + "Me" -> LazyCategory(CharCategory.ENCLOSING_MARK.value, false) + "Mc" -> LazyCategory(CharCategory.COMBINING_SPACING_MARK.value, true) + "N" -> LazyCategoryScope(0xE00, true) + "Nd" -> LazyCategory(CharCategory.DECIMAL_DIGIT_NUMBER.value, true) + "Nl" -> LazyCategory(CharCategory.LETTER_NUMBER.value, true) + "No" -> LazyCategory(CharCategory.OTHER_NUMBER.value, true) + "IsZ" -> LazyCategoryScope(0x7000, false) + "Zs" -> LazyCategory(CharCategory.SPACE_SEPARATOR.value, false) + "Zl" -> LazyCategory(CharCategory.LINE_SEPARATOR.value, false) + "Zp" -> LazyCategory(CharCategory.PARAGRAPH_SEPARATOR.value, false) + "IsC" -> LazyCategoryScope(0xF0000, true, true) + "Cc" -> LazyCategory(CharCategory.CONTROL.value, false) + "Cf" -> LazyCategory(CharCategory.FORMAT.value, true) + "Co" -> LazyCategory(CharCategory.PRIVATE_USE.value, true) + "Cs" -> LazyCategory(CharCategory.SURROGATE.value, false, true) + "IsP" -> LazyCategoryScope(1 shl CharCategory.DASH_PUNCTUATION.value or + (1 shl CharCategory.START_PUNCTUATION.value) or + (1 shl CharCategory.END_PUNCTUATION.value) or + (1 shl CharCategory.CONNECTOR_PUNCTUATION.value) or + (1 shl CharCategory.OTHER_PUNCTUATION.value) or + (1 shl CharCategory.INITIAL_QUOTE_PUNCTUATION.value) or + (1 shl CharCategory.FINAL_QUOTE_PUNCTUATION.value), true) + "Pd" -> LazyCategory(CharCategory.DASH_PUNCTUATION.value, false) + "Ps" -> LazyCategory(CharCategory.START_PUNCTUATION.value, false) + "Pe" -> LazyCategory(CharCategory.END_PUNCTUATION.value, false) + "Pc" -> LazyCategory(CharCategory.CONNECTOR_PUNCTUATION.value, false) + "Po" -> LazyCategory(CharCategory.OTHER_PUNCTUATION.value, true) + "IsS" -> LazyCategoryScope(0x7E000000, true) + "Sm" -> LazyCategory(CharCategory.MATH_SYMBOL.value, true) + "Sc" -> LazyCategory(CharCategory.CURRENCY_SYMBOL.value, false) + "Sk" -> LazyCategory(CharCategory.MODIFIER_SYMBOL.value, false) + "So" -> LazyCategory(CharCategory.OTHER_SYMBOL.value, true) + "Pi" -> LazyCategory(CharCategory.INITIAL_QUOTE_PUNCTUATION.value, false) + "Pf" -> LazyCategory(CharCategory.FINAL_QUOTE_PUNCTUATION.value, false) + else -> throw PatternSyntaxException("No such character class") + } + } + + fun intersects(ch1: Int, ch2: Int): Boolean { + return ch1 == ch2 + } + + fun intersects(cc: AbstractCharClass, ch: Int): Boolean { + return cc.contains(ch) + } + + fun intersects(cc1: AbstractCharClass, + cc2: AbstractCharClass): Boolean { + if (cc1.bits == null || cc2.bits == null) + return true + return cc1.bits!!.intersects(cc2.bits!!) + } + + fun getPredefinedClass(name: String, negative: Boolean): AbstractCharClass { + return (getClass(name)).getValue(negative) + } + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/AbstractLineTerminator.kt b/runtime/src/main/kotlin/kotlin/text/regex/AbstractLineTerminator.kt new file mode 100644 index 00000000000..c076f714ed1 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/AbstractLineTerminator.kt @@ -0,0 +1,59 @@ +// TODO: License. + +package kotlin.text + +/** + * Line terminator factory + * + * @author Nikolay A. Kuznetsov + */ +internal abstract class AbstractLineTerminator { + + /** Checks if the single character is a line terminator or not. */ + open fun isLineTerminator(char: Char): Boolean = isLineTerminator(char.toInt()) + /** Checks if the codepoint is a line terminator or not */ + abstract fun isLineTerminator(codepoint: Int): Boolean + /** Checks if the pair of symbols is a line terminator (e.g. for \r\n case) */ + abstract fun isLineTerminatorPair(char1: Char, char2: Char): Boolean + /** Checks if a [checked] character is after a line terminator using the [previous] character.*/ + abstract fun isAfterLineTerminator(previous: Char, checked: Char): Boolean + + companion object { + val unixLT: AbstractLineTerminator by lazy { + object : AbstractLineTerminator() { + override fun isLineTerminator(codepoint: Int): Boolean = (codepoint == '\n'.toInt()) + override fun isLineTerminatorPair(char1: Char, char2: Char): Boolean = false + override fun isAfterLineTerminator(previous: Char, checked: Char): Boolean = (previous == '\n') + } + } + + val unicodeLT: AbstractLineTerminator by lazy { + object : AbstractLineTerminator() { + override fun isLineTerminatorPair(char1: Char, char2: Char): Boolean { + return char1 == '\r' && char2 == '\n' + } + + override fun isLineTerminator(codepoint: Int): Boolean { + return codepoint == '\n'.toInt() + || codepoint == '\r'.toInt() + || codepoint == '\u0085'.toInt() + || codepoint or 1 == '\u2029'.toInt() + } + + override fun isAfterLineTerminator(previous: Char, checked: Char): Boolean { + return previous == '\n' || previous == '\u0085' || previous.toInt() or 1 == '\u2029'.toInt() + || previous == '\r' && checked != '\n' + } + } + } + + // TODO: Replace with property or another Kotlin-way singleton. + fun getInstance(flag: Int): AbstractLineTerminator { + if (flag and Pattern.UNIX_LINES != 0) { + return unixLT + } else { + return unicodeLT + } + } + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/CharClass.kt b/runtime/src/main/kotlin/kotlin/text/regex/CharClass.kt new file mode 100644 index 00000000000..06d3d5bc2fd --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/CharClass.kt @@ -0,0 +1,547 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * User defined character classes (e.g. [abef]). + * + * @author Nikolay A. Kuznetsov + */ +/* + * TODO: replace the implementation with one using BitSet for first 256 symbols and a hash table / tree for the rest of UTF. + */ +internal class CharClass(val ignoreCase: Boolean = false, negative: Boolean = false) : AbstractCharClass() { + + var invertedSurrogates = false + + /** + * Shows if the alt flags was inverted during the range construction process. + * E.g. consider the following range: [\D3]. Here we firstly add the \D char class (which has the alt flag set) into a + * resulting char set. After that the resulting char also has the alt flag set. But then we need to add the '3' character + * into this class with positive sense. So we set the inverted flag to show that the range is not negative ([^..]) + * by itself but was inverted during some transformations. + */ + var inverted = false + + // TODO: May be we can get rid of it? + var hideBits = false + + internal var bits_ = BitSet() + override val bits: BitSet? + get() { + if (hideBits) + return null + return bits_ + } + + var nonBitSet: AbstractCharClass? = null + + private val Int.asciiSupplement: Int + get() = when { + this in 'a'.toInt()..'z'.toInt() -> this - 32 + this in 'A'.toInt()..'Z'.toInt() -> this + 32 + else -> this + } + private val Int.isSurrogate: Boolean + get() = this in Char.MIN_SURROGATE.toInt()..Char.MAX_SURROGATE.toInt() + + init { + setNegative(negative) + } + + /* + * We can use this method safely even if nonBitSet != null + * due to specific of range constructions in regular expressions. + */ + fun add(ch: Int): CharClass { + var character = ch + if (ignoreCase) { + if (character.toChar() in 'a'..'z' || character.toChar() in 'A'..'Z') { + bits_.set(character.asciiSupplement, !inverted) + } else if (character > 128) { + character = Char.toLowerCase(Char.toUpperCase(character)) + } + } + if (character.toChar().isSurrogate()) { + lowHighSurrogates.set(character - Char.MIN_SURROGATE.toInt(), !invertedSurrogates) + } + bits_.set(character, !inverted) + if (!mayContainSupplCodepoints && Char.isSupplementaryCodePoint(ch)) { + mayContainSupplCodepoints = true + } + return this + } + + fun add(ch: Char): CharClass = add(ch.toInt()) + + /* + * The difference between add(AbstractCharClass) and union(AbstractCharClass) + * is that add() is used for constructions like "[^abc\\d]" + * (this pattern doesn't match "1") + * while union is used for constructions like "[^abc[\\d]]" + * (this pattern matches "1"). + */ + fun add(another: AbstractCharClass): CharClass { + + if (!mayContainSupplCodepoints && another.mayContainSupplCodepoints) { + mayContainSupplCodepoints = true + } + + // Process surrogates. + if (!invertedSurrogates) { + + //A | !B = ! ((A ^ B) & B) + if (another.altSurrogates) { + lowHighSurrogates.xor(another.lowHighSurrogates) + lowHighSurrogates.and(another.lowHighSurrogates) + altSurrogates = !altSurrogates + invertedSurrogates = true + + //A | B + } else { + lowHighSurrogates.or(another.lowHighSurrogates) + } + } else { + + //!A | !B = !(A & B) + if (another.altSurrogates) { + lowHighSurrogates.and(another.lowHighSurrogates) + + //!A | B = !(A & !B) + } else { + lowHighSurrogates.andNot(another.lowHighSurrogates) + } + } + + val anotherBits = another.bits + if (!hideBits && anotherBits != null) { + if (!inverted) { + + //A | !B = ! ((A ^ B) & B) + if (another.isNegative()) { + bits_.xor(anotherBits) + bits_.and(anotherBits) + alt = !alt + inverted = true + + //A | B + } else { + bits_.or(anotherBits) + } + } else { + + //!A | !B = !(A & B) + if (another.isNegative()) { + bits_.and(anotherBits) + + //!A | B = !(A & !B) + } else { + bits_.andNot(anotherBits) + } + } + // Some of charclasses hides its bits + // Looks like hideBits and nonBitSet are used when we add another class which have no bitmask. + // TODO: We can potentially remove nonBitSet and hideBits. What is alt? + // TODO: The same for or operation. + } else { + val curAlt = alt + + if (nonBitSet == null) { + + if (curAlt && !inverted && bits_.isEmpty) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return another.contains(ch) + } + } + //alt = true; + } else { + + /* + * We keep the value of alt unchanged for + * constructions like [^[abc]fgb] by using + * the formula a ^ b == !a ^ !b. + */ + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !(curAlt xor bits_.get(ch) || curAlt xor inverted xor another.contains(ch)) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return curAlt xor bits_.get(ch) || curAlt xor inverted xor another.contains(ch) + } + } + //alt = false + } + } + + hideBits = true + } else { + val nb = nonBitSet + + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !(curAlt xor (nb!!.contains(ch) || another.contains(ch))) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return curAlt xor (nb!!.contains(ch) || another.contains(ch)) + } + } + //alt = false + } + } + } + + return this + } + + fun add(start: Int, end: Int): CharClass { + if (start > end) + throw IllegalArgumentException() + //no intersection with surrogate characters + if (!ignoreCase && (end < Char.MIN_SURROGATE.toInt() || start > Char.MAX_SURROGATE.toInt())) { + if (!inverted) { + bits_.set(start, end + 1) + } else { + bits_.clear(start, end + 1) + } + } else { + for (i in start..end) { + add(i) + } + } + return this + } + + fun add(start: Char, end: Char): CharClass = add(start.toInt(), end.toInt()) + + // OR operation + fun union(another: AbstractCharClass) { + if (!mayContainSupplCodepoints && another.mayContainSupplCodepoints) { + mayContainSupplCodepoints = true + } + + + if (altSurrogates xor another.altSurrogates) { + + //!A | B = !(A & !B) + if (altSurrogates) { + lowHighSurrogates.andNot(another.lowHighSurrogates) + + //A | !B = !((A ^ B) & B) + } else { + lowHighSurrogates.xor(another.lowHighSurrogates) + lowHighSurrogates.and(another.lowHighSurrogates) + altSurrogates = true + } + + } else { + + //!A | !B = !(A & B) + if (altSurrogates) { + lowHighSurrogates.and(another.lowHighSurrogates) + + //A | B + } else { + lowHighSurrogates.or(another.lowHighSurrogates) + } + } + + val anotherBits = another.bits + if (!hideBits && anotherBits != null) { + if (alt xor another.isNegative()) { + + //!A | B = !(A & !B) + if (alt) { + bits_.andNot(anotherBits) + + //A | !B = !((A ^ B) & B) + } else { + bits_.xor(anotherBits) + bits_.and(anotherBits) + alt = true + } + + } else { + + //!A | !B = !(A & B) + if (alt) { + bits_.and(anotherBits) + + //A | B + } else { + bits_.or(anotherBits) + } + } + } else { + val curAlt = alt + + if (nonBitSet == null) { + + if (!inverted && bits_.isEmpty) { + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !another.contains(ch) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return another.contains(ch) + } + } + //alt = false + } + } else { + + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !(another.contains(ch) || curAlt xor bits_.get(ch)) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return another.contains(ch) || curAlt xor bits_.get(ch) + } + } + //alt = false + } + } + hideBits = true + } else { + val nb = nonBitSet + + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !(curAlt xor nb!!.contains(ch) || another.contains(ch)) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return curAlt xor nb!!.contains(ch) || another.contains(ch) + } + } + //alt = false + } + } + } + } + + // AND operation + fun intersection(another: AbstractCharClass) { + if (!mayContainSupplCodepoints && another.mayContainSupplCodepoints) { + mayContainSupplCodepoints = true + } + + if (altSurrogates xor another.altSurrogates) { + + //!A & B = ((A ^ B) & B) + if (altSurrogates) { + lowHighSurrogates.xor(another.lowHighSurrogates) + lowHighSurrogates.and(another.lowHighSurrogates) + altSurrogates = false + + //A & !B + } else { + lowHighSurrogates.andNot(another.lowHighSurrogates) + } + } else { + + //!A & !B = !(A | B) + if (altSurrogates) { + lowHighSurrogates.or(another.lowHighSurrogates) + + //A & B + } else { + lowHighSurrogates.and(another.lowHighSurrogates) + } + } + + val anotherBits = another.bits + if (!hideBits && anotherBits != null) { + + if (alt xor another.isNegative()) { + + //!A & B = ((A ^ B) & B) + if (alt) { + bits_.xor(anotherBits) + bits_.and(anotherBits) + alt = false + + //A & !B + } else { + bits_.andNot(anotherBits) + } + } else { + + //!A & !B = !(A | B) + if (alt) { + bits_.or(anotherBits) + + //A & B + } else { + bits_.and(anotherBits) + } + } + } else { + val curAlt = alt + + if (nonBitSet == null) { + + if (!inverted && bits_.isEmpty) { + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !another.contains(ch) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return another.contains(ch) + } + } + //alt = false + } + } else { + + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !(another.contains(ch) && curAlt xor bits_.get(ch)) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return another.contains(ch) && curAlt xor bits_.get(ch) + } + } + //alt = false + } + } + hideBits = true + } else { + val nb = nonBitSet + + if (curAlt) { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return !(curAlt xor nb!!.contains(ch) && another.contains(ch)) + } + } + //alt = true + } else { + nonBitSet = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return curAlt xor nb!!.contains(ch) && another.contains(ch) + } + } + //alt = false + } + } + } + } + + /** + * Returns `true` if character class contains symbol specified, + * `false` otherwise. Note: #setNegative() method changes the + * meaning of contains method; + + * @param ch + * * + * @return `true` if character class contains symbol specified; + * * + */ + override operator fun contains(ch: Int): Boolean { + if (nonBitSet == null) { + return this.alt xor bits_.get(ch) // TODO alt xor bits. It must make sense. + } else { + return alt xor nonBitSet!!.contains(ch) + } + } + + override val instance: AbstractCharClass + get() { + + if (nonBitSet == null) { + val bs = bits + + val res = object : AbstractCharClass() { + override operator fun contains(ch: Int): Boolean { + return this.alt xor bs!!.get(ch) + } + + override fun toString(): String { + val temp = StringBuilder() + var i = bs!!.nextSetBit(0) + while (i >= 0) { + temp.append(Char.toChars(i)) + temp.append('|') + i = bs.nextSetBit(i + 1) + } + + if (temp.length > 0) + temp.deleteCharAt(temp.length - 1) + + return temp.toString() + } + + } + return res.setNegative(isNegative()) + } else { + return this + } + } + + //for debugging purposes only + override fun toString(): String { + val temp = StringBuilder() + var i = bits_.nextSetBit(0) + while (i >= 0) { + temp.append(Char.toChars(i)) + temp.append('|') + i = bits_.nextSetBit(i + 1) + } + + if (temp.length > 0) + temp.deleteCharAt(temp.length - 1) + + return temp.toString() + } +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/Decompositions.kt b/runtime/src/main/kotlin/kotlin/text/regex/Decompositions.kt new file mode 100644 index 00000000000..ee244e4b831 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/Decompositions.kt @@ -0,0 +1,3406 @@ +// TODO: @C++. Create tables on the C++ side + +// TODO: Theoretically we can normalize the string using separate C++ unicode lib + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * This class gives us a hashtable that contains canonical + * classes that are generated from + * http://www.unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt. + */ +internal object CanClasses { + val hashCanClasses: IntHash by lazy { + val table = IntHash(384) + table.put(0x20D0, 230) + table.put(0x0A4D, 9) + table.put(0x0E49, 107) + table.put(0x0954, 230) + table.put(0x0E48, 107) + table.put(0x0953, 230) + table.put(0x0952, 220) + table.put(0x0951, 230) + table.put(0x094D, 9) + table.put(0x0E3A, 9) + table.put(0x17DD, 230) + table.put(0x0A3C, 7) + table.put(0x0E39, 103) + table.put(0x0E38, 103) + table.put(0x17D2, 9) + table.put(0x093C, 7) + table.put(0x06ED, 220) + table.put(0x06EC, 230) + table.put(0x06EB, 230) + table.put(0x06EA, 220) + table.put(0x06E8, 230) + table.put(0x06E7, 230) + table.put(0x06E4, 230) + table.put(0x06E3, 220) + table.put(0x06E2, 230) + table.put(0x06E1, 230) + table.put(0x06E0, 230) + table.put(0x06DF, 230) + table.put(0x06DC, 230) + table.put(0x06DB, 230) + table.put(0x06DA, 230) + table.put(0x0BCD, 9) + table.put(0x0486, 230) + table.put(0x0485, 230) + table.put(0x0484, 230) + table.put(0x0FC6, 220) + table.put(0x0483, 230) + table.put(0x06D9, 230) + table.put(0x06D8, 230) + table.put(0x06D7, 230) + table.put(0x06D6, 230) + table.put(0xA806, 9) + table.put(0x193B, 220) + table.put(0x193A, 230) + table.put(0x1939, 222) + table.put(0x0D4D, 9) + table.put(0x1A18, 220) + table.put(0x1A17, 230) + table.put(0x1D1AD, 230) + table.put(0x1D1AC, 230) + table.put(0x1D1AB, 230) + table.put(0x1D1AA, 230) + table.put(0xFB1E, 26) + table.put(0x0ECB, 122) + table.put(0x0ECA, 122) + table.put(0x0ACD, 9) + table.put(0x0EC9, 122) + table.put(0x0EC8, 122) + table.put(0x09CD, 9) + table.put(0x0ABC, 7) + table.put(0x0EB9, 118) + table.put(0x0EB8, 118) + table.put(0x05C7, 18) + table.put(0x05C5, 220) + table.put(0x05C4, 230) + table.put(0x05C2, 25) + table.put(0x05C1, 24) + table.put(0x036F, 230) + table.put(0x036E, 230) + table.put(0x09BC, 7) + table.put(0x036D, 230) + table.put(0x036C, 230) + table.put(0x036B, 230) + table.put(0x036A, 230) + table.put(0x05BF, 23) + table.put(0x05BD, 22) + table.put(0x05BC, 21) + table.put(0x05BB, 20) + table.put(0x0C56, 91) + table.put(0x0C55, 84) + table.put(0x0369, 230) + table.put(0x0368, 230) + table.put(0x0367, 230) + table.put(0x0366, 230) + table.put(0x0365, 230) + table.put(0x0364, 230) + table.put(0x0363, 230) + table.put(0x0362, 233) + table.put(0x05B9, 19) + table.put(0x0361, 234) + table.put(0x05B8, 18) + table.put(0x0360, 234) + table.put(0x05B7, 17) + table.put(0x05B6, 16) + table.put(0x05B5, 15) + table.put(0x05B4, 14) + table.put(0x05B3, 13) + table.put(0x05B2, 12) + table.put(0x05B1, 11) + table.put(0x0C4D, 9) + table.put(0x05B0, 10) + table.put(0x035F, 233) + table.put(0x035E, 234) + table.put(0x035D, 234) + table.put(0x035C, 233) + table.put(0x035B, 230) + table.put(0x035A, 220) + table.put(0x05AF, 230) + table.put(0x05AE, 228) + table.put(0x05AD, 222) + table.put(0x05AC, 230) + table.put(0x05AB, 230) + table.put(0x05AA, 220) + table.put(0x1039, 9) + table.put(0x0359, 220) + table.put(0x0358, 232) + table.put(0x1037, 7) + table.put(0x0357, 230) + table.put(0x0356, 220) + table.put(0x0355, 220) + table.put(0x0354, 220) + table.put(0x0353, 220) + table.put(0x0352, 230) + table.put(0x05A9, 230) + table.put(0x0351, 230) + table.put(0x05A8, 230) + table.put(0x0350, 230) + table.put(0x05A7, 220) + table.put(0x05A6, 220) + table.put(0x05A5, 220) + table.put(0x05A4, 220) + table.put(0x05A3, 220) + table.put(0x05A2, 220) + table.put(0x074A, 230) + table.put(0x05A1, 230) + table.put(0x05A0, 230) + table.put(0x034E, 220) + table.put(0x034D, 220) + table.put(0x034C, 230) + table.put(0x034B, 230) + table.put(0x0749, 230) + table.put(0x034A, 230) + table.put(0x0748, 220) + table.put(0x0747, 230) + table.put(0x0746, 220) + table.put(0x0745, 230) + table.put(0x0744, 220) + table.put(0x0743, 230) + table.put(0x0742, 220) + table.put(0x0741, 230) + table.put(0x0349, 220) + table.put(0x0740, 230) + table.put(0x0348, 220) + table.put(0x0347, 220) + table.put(0x0346, 230) + table.put(0x0345, 240) + table.put(0x0344, 230) + table.put(0x0343, 230) + table.put(0x0342, 230) + table.put(0x0341, 230) + table.put(0x0340, 230) + table.put(0x073F, 230) + table.put(0x073E, 220) + table.put(0x073D, 230) + table.put(0x073C, 220) + table.put(0x073B, 220) + table.put(0x073A, 230) + table.put(0x309A, 8) + table.put(0x033F, 230) + table.put(0x033E, 230) + table.put(0x033D, 230) + table.put(0x033C, 220) + table.put(0x033B, 220) + table.put(0x0739, 220) + table.put(0x033A, 220) + table.put(0x0738, 220) + table.put(0x0737, 220) + table.put(0x0736, 230) + table.put(0x3099, 8) + table.put(0x0735, 230) + table.put(0xFE23, 230) + table.put(0x0734, 220) + table.put(0x0F87, 230) + table.put(0xFE22, 230) + table.put(0x0733, 230) + table.put(0x0F86, 230) + table.put(0xFE21, 230) + table.put(0x0732, 230) + table.put(0xFE20, 230) + table.put(0x0731, 220) + table.put(0x0F84, 9) + table.put(0x0339, 220) + table.put(0x0730, 230) + table.put(0x0F83, 230) + table.put(0x0338, 1) + table.put(0x0F82, 230) + table.put(0x0337, 1) + table.put(0x0336, 1) + table.put(0x0F80, 130) + table.put(0x0335, 1) + table.put(0x0334, 1) + table.put(0x0333, 220) + table.put(0x0332, 220) + table.put(0x0331, 220) + table.put(0x0330, 220) + table.put(0x1D244, 230) + table.put(0x1D243, 230) + table.put(0x1D242, 230) + table.put(0x0F7D, 130) + table.put(0x0F7C, 130) + table.put(0x0F7B, 130) + table.put(0x0F7A, 130) + table.put(0x032F, 220) + table.put(0x032E, 220) + table.put(0x032D, 220) + table.put(0x032C, 220) + table.put(0x032B, 220) + table.put(0x032A, 220) + table.put(0x0F74, 132) + table.put(0x0329, 220) + table.put(0x0328, 202) + table.put(0x0F72, 130) + table.put(0x0327, 202) + table.put(0x0DCA, 9) + table.put(0x0F71, 129) + table.put(0x0326, 220) + table.put(0x0325, 220) + table.put(0x0324, 220) + table.put(0x0323, 220) + table.put(0x0322, 202) + table.put(0x0321, 202) + table.put(0x0320, 220) + table.put(0x10A3F, 9) + table.put(0x135F, 230) + table.put(0x10A3A, 220) + table.put(0x031F, 220) + table.put(0x031E, 220) + table.put(0x031D, 220) + table.put(0x031C, 220) + table.put(0x031B, 216) + table.put(0x031A, 232) + table.put(0x10A39, 1) + table.put(0x10A38, 230) + table.put(0x0711, 36) + table.put(0x0319, 220) + table.put(0x0318, 220) + table.put(0x0317, 220) + table.put(0x0316, 220) + table.put(0x0315, 232) + table.put(0x0314, 230) + table.put(0x1D18B, 220) + table.put(0x0313, 230) + table.put(0x1D18A, 220) + table.put(0x0312, 230) + table.put(0x0311, 230) + table.put(0x0670, 35) + table.put(0x0310, 230) + table.put(0x1D189, 230) + table.put(0x1D188, 230) + table.put(0x1D187, 230) + table.put(0x1D186, 230) + table.put(0x030F, 230) + table.put(0x1D185, 230) + table.put(0x030E, 230) + table.put(0x030D, 230) + table.put(0x030C, 230) + table.put(0x1D182, 220) + table.put(0x030B, 230) + table.put(0x1D181, 220) + table.put(0x030A, 230) + table.put(0x1D180, 220) + table.put(0x0309, 230) + table.put(0x0308, 230) + table.put(0x1D17F, 220) + table.put(0x0307, 230) + table.put(0x1D17E, 220) + table.put(0x0306, 230) + table.put(0x1D17D, 220) + table.put(0x0305, 230) + table.put(0x1D17C, 220) + table.put(0x0304, 230) + table.put(0x1D17B, 220) + table.put(0x0303, 230) + table.put(0x0302, 230) + table.put(0x0301, 230) + table.put(0x0300, 230) + table.put(0x065E, 230) + table.put(0x065D, 230) + table.put(0x065C, 220) + table.put(0x065B, 230) + table.put(0x1D172, 216) + table.put(0x065A, 230) + table.put(0x1D171, 216) + table.put(0x0B4D, 9) + table.put(0x1D170, 216) + table.put(0x1734, 9) + table.put(0x0659, 230) + table.put(0x0658, 230) + table.put(0x0657, 230) + table.put(0x1D16F, 216) + table.put(0x0656, 220) + table.put(0x1D16E, 216) + table.put(0x0655, 220) + table.put(0x1D16D, 226) + table.put(0x0654, 230) + table.put(0x0653, 230) + table.put(0x0652, 34) + table.put(0x0651, 33) + table.put(0x0650, 32) + table.put(0x10A0F, 230) + table.put(0x10A0D, 220) + table.put(0x1D169, 1) + table.put(0x1D168, 1) + table.put(0x1D167, 1) + table.put(0x064F, 31) + table.put(0x1D166, 216) + table.put(0x064E, 30) + table.put(0x1D165, 216) + table.put(0x064D, 29) + table.put(0x064C, 28) + table.put(0x064B, 27) + table.put(0x0B3C, 7) + table.put(0x0F39, 216) + table.put(0x0F37, 220) + table.put(0x0F35, 220) + table.put(0x1DC3, 230) + table.put(0x1DC2, 220) + table.put(0x1DC1, 230) + table.put(0x1DC0, 230) + table.put(0x059F, 230) + table.put(0x1714, 9) + table.put(0x059E, 230) + table.put(0x059D, 230) + table.put(0x059C, 230) + table.put(0x059B, 220) + table.put(0x059A, 222) + table.put(0x0599, 230) + table.put(0x0598, 230) + table.put(0x0597, 230) + table.put(0x0596, 220) + table.put(0x0595, 230) + table.put(0x0594, 230) + table.put(0x0593, 230) + table.put(0x302F, 224) + table.put(0x0592, 230) + table.put(0x302E, 224) + table.put(0x0591, 220) + table.put(0x302D, 222) + table.put(0x302C, 232) + table.put(0x302B, 228) + table.put(0x302A, 218) + table.put(0x0F19, 220) + table.put(0x0F18, 220) + table.put(0x0CCD, 9) + table.put(0x0615, 230) + table.put(0x0614, 230) + table.put(0x18A9, 228) + table.put(0x0613, 230) + table.put(0x0612, 230) + table.put(0x0611, 230) + table.put(0x0CBC, 7) + table.put(0x0610, 230) + table.put(0x20EB, 1) + table.put(0x20EA, 1) + table.put(0x20E9, 230) + table.put(0x20E8, 220) + table.put(0x20E7, 230) + table.put(0x20E6, 1) + table.put(0x20E5, 1) + table.put(0x20E1, 230) + table.put(0x20DC, 230) + table.put(0x20DB, 230) + table.put(0x20DA, 1) + table.put(0x20D9, 1) + table.put(0x20D8, 1) + table.put(0x20D7, 230) + table.put(0x20D6, 230) + table.put(0x0E4B, 107) + table.put(0x20D5, 230) + table.put(0x0E4A, 107) + table.put(0x20D4, 230) + table.put(0x20D3, 1) + table.put(0x20D2, 1) + table.put(0x20D1, 230) + return@lazy table + } +} +/** + * This class gives us a hashtable that contains canonical + * decomposition mappings that are generated from + * http://www.unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt. + */ +internal object HashDecompositions { + val hashDecompositions: IntArrHash by lazy { + val table = IntArrHash(2032) + table.put(0x0929, intArrayOf(0x0928, 0x093C)) + table.put(0xFA9F, intArrayOf(0x72AF)) + table.put(0xFA9E, intArrayOf(0x7235)) + table.put(0xFA9D, intArrayOf(0x77A7)) + table.put(0xFA9C, intArrayOf(0x716E)) + table.put(0xFA9B, intArrayOf(0x701E)) + table.put(0xFA9A, intArrayOf(0x6F22)) + table.put(0x0BCC, intArrayOf(0x0BC6, 0x0BD7)) + table.put(0x0BCB, intArrayOf(0x0BC7, 0x0BBE)) + table.put(0x0BCA, intArrayOf(0x0BC6, 0x0BBE)) + table.put(0xFA99, intArrayOf(0x6ECB)) + table.put(0xFA98, intArrayOf(0x6EDB)) + table.put(0xFA97, intArrayOf(0x6D41)) + table.put(0xFA96, intArrayOf(0x6BBA)) + table.put(0xFA95, intArrayOf(0x6B79)) + table.put(0xFA94, intArrayOf(0x6756)) + table.put(0xFA93, intArrayOf(0x671B)) + table.put(0xFA92, intArrayOf(0x6717)) + table.put(0x1D164, intArrayOf(0x1D158, 0x1D165, 0x1D172)) + table.put(0xFA91, intArrayOf(0x6674)) + table.put(0x1D163, intArrayOf(0x1D158, 0x1D165, 0x1D171)) + table.put(0xFA90, intArrayOf(0x6556)) + table.put(0x1D162, intArrayOf(0x1D158, 0x1D165, 0x1D170)) + table.put(0x1D161, intArrayOf(0x1D158, 0x1D165, 0x1D16F)) + table.put(0x1D160, intArrayOf(0x1D158, 0x1D165, 0x1D16E)) + table.put(0xFA8F, intArrayOf(0x6452)) + table.put(0xFA8E, intArrayOf(0x641C)) + table.put(0xFA8D, intArrayOf(0x63C4)) + table.put(0xFA8C, intArrayOf(0x6234)) + table.put(0x1D15F, intArrayOf(0x1D158, 0x1D165)) + table.put(0xFA8B, intArrayOf(0x61F2)) + table.put(0x1D15E, intArrayOf(0x1D157, 0x1D165)) + table.put(0xFA8A, intArrayOf(0x6160)) + table.put(0xFA89, intArrayOf(0x618E)) + table.put(0xFA88, intArrayOf(0x6108)) + table.put(0xFA87, intArrayOf(0x614E)) + table.put(0xFA86, intArrayOf(0x60D8)) + table.put(0xFA85, intArrayOf(0x5FAD)) + table.put(0xFA84, intArrayOf(0x5F69)) + table.put(0xFA83, intArrayOf(0x5ED9)) + table.put(0xFA82, intArrayOf(0x5ED2)) + table.put(0xFA81, intArrayOf(0x5B28)) + table.put(0xFA80, intArrayOf(0x5A62)) + table.put(0xFA7F, intArrayOf(0x5954)) + table.put(0x1FFD, intArrayOf(0x00B4)) + table.put(0xFA7E, intArrayOf(0x5944)) + table.put(0x1FFC, intArrayOf(0x03A9, 0x0345)) + table.put(0xFA7D, intArrayOf(0x58B3)) + table.put(0x1FFB, intArrayOf(0x03A9, 0x0301)) + table.put(0xFA7C, intArrayOf(0x585A)) + table.put(0x1FFA, intArrayOf(0x03A9, 0x0300)) + table.put(0xFA7B, intArrayOf(0x55E2)) + table.put(0xFA7A, intArrayOf(0x5599)) + table.put(0x219B, intArrayOf(0x2192, 0x0338)) + table.put(0x219A, intArrayOf(0x2190, 0x0338)) + table.put(0x1FF9, intArrayOf(0x039F, 0x0301)) + table.put(0x1FF8, intArrayOf(0x039F, 0x0300)) + table.put(0xFA79, intArrayOf(0x5555)) + table.put(0x1FF7, intArrayOf(0x03C9, 0x0342, 0x0345)) + table.put(0xFA78, intArrayOf(0x559D)) + table.put(0x1FF6, intArrayOf(0x03C9, 0x0342)) + table.put(0xFA77, intArrayOf(0x52FA)) + table.put(0xFA76, intArrayOf(0x52C7)) + table.put(0x1FF4, intArrayOf(0x03C9, 0x0301, 0x0345)) + table.put(0xFA75, intArrayOf(0x5180)) + table.put(0x1FF3, intArrayOf(0x03C9, 0x0345)) + table.put(0xFA74, intArrayOf(0x5145)) + table.put(0x1FF2, intArrayOf(0x03C9, 0x0300, 0x0345)) + table.put(0xFA73, intArrayOf(0x4F80)) + table.put(0xFA72, intArrayOf(0x5168)) + table.put(0xFA71, intArrayOf(0x51B5)) + table.put(0xFA70, intArrayOf(0x4E26)) + table.put(0x1FEF, intArrayOf(0x0060)) + table.put(0x1FEE, intArrayOf(0x00A8, 0x0301)) + table.put(0x1FED, intArrayOf(0x00A8, 0x0300)) + table.put(0x1FEC, intArrayOf(0x03A1, 0x0314)) + table.put(0x1FEB, intArrayOf(0x03A5, 0x0301)) + table.put(0x1FEA, intArrayOf(0x03A5, 0x0300)) + table.put(0xFA6A, intArrayOf(0x983B)) + table.put(0x1FE9, intArrayOf(0x03A5, 0x0304)) + table.put(0x1FE8, intArrayOf(0x03A5, 0x0306)) + table.put(0xFA69, intArrayOf(0x97FF)) + table.put(0x1FE7, intArrayOf(0x03C5, 0x0308, 0x0342)) + table.put(0xFA68, intArrayOf(0x96E3)) + table.put(0x1FE6, intArrayOf(0x03C5, 0x0342)) + table.put(0xFA67, intArrayOf(0x9038)) + table.put(0x1FE5, intArrayOf(0x03C1, 0x0314)) + table.put(0xFA66, intArrayOf(0x8FB6)) + table.put(0x1FE4, intArrayOf(0x03C1, 0x0313)) + table.put(0xFA65, intArrayOf(0x8D08)) + table.put(0x1FE3, intArrayOf(0x03C5, 0x0308, 0x0301)) + table.put(0xFA64, intArrayOf(0x8CD3)) + table.put(0x1FE2, intArrayOf(0x03C5, 0x0308, 0x0300)) + table.put(0xFA63, intArrayOf(0x8B39)) + table.put(0x1FE1, intArrayOf(0x03C5, 0x0304)) + table.put(0xFA62, intArrayOf(0x8B01)) + table.put(0x1FE0, intArrayOf(0x03C5, 0x0306)) + table.put(0xFA61, intArrayOf(0x8996)) + table.put(0xFA60, intArrayOf(0x8910)) + table.put(0x1FDF, intArrayOf(0x1FFE, 0x0342)) + table.put(0x1FDE, intArrayOf(0x1FFE, 0x0301)) + table.put(0xFA5F, intArrayOf(0x8457)) + table.put(0x1FDD, intArrayOf(0x1FFE, 0x0300)) + table.put(0xFA5E, intArrayOf(0x8279)) + table.put(0xFA5D, intArrayOf(0x8279)) + table.put(0x1FDB, intArrayOf(0x0399, 0x0301)) + table.put(0xFA5C, intArrayOf(0x81ED)) + table.put(0x1FDA, intArrayOf(0x0399, 0x0300)) + table.put(0xFA5B, intArrayOf(0x8005)) + table.put(0xFA5A, intArrayOf(0x7F72)) + table.put(0x1FD9, intArrayOf(0x0399, 0x0304)) + table.put(0x1FD8, intArrayOf(0x0399, 0x0306)) + table.put(0xFA59, intArrayOf(0x7E41)) + table.put(0x1FD7, intArrayOf(0x03B9, 0x0308, 0x0342)) + table.put(0xFA58, intArrayOf(0x7E09)) + table.put(0x1FD6, intArrayOf(0x03B9, 0x0342)) + table.put(0xFA57, intArrayOf(0x7DF4)) + table.put(0xFA56, intArrayOf(0x7BC0)) + table.put(0xFA55, intArrayOf(0x7A81)) + table.put(0x1FD3, intArrayOf(0x03B9, 0x0308, 0x0301)) + table.put(0xFA54, intArrayOf(0x7A40)) + table.put(0x1FD2, intArrayOf(0x03B9, 0x0308, 0x0300)) + table.put(0xFA53, intArrayOf(0x798E)) + table.put(0x1FD1, intArrayOf(0x03B9, 0x0304)) + table.put(0xFA52, intArrayOf(0x798D)) + table.put(0x1FD0, intArrayOf(0x03B9, 0x0306)) + table.put(0xFA51, intArrayOf(0x795D)) + table.put(0xFA50, intArrayOf(0x7956)) + table.put(0x1FCF, intArrayOf(0x1FBF, 0x0342)) + table.put(0x1FCE, intArrayOf(0x1FBF, 0x0301)) + table.put(0xFA4F, intArrayOf(0x7950)) + table.put(0x1FCD, intArrayOf(0x1FBF, 0x0300)) + table.put(0xFA4E, intArrayOf(0x7948)) + table.put(0x1FCC, intArrayOf(0x0397, 0x0345)) + table.put(0xFA4D, intArrayOf(0x7949)) + table.put(0x1FCB, intArrayOf(0x0397, 0x0301)) + table.put(0xFA4C, intArrayOf(0x793E)) + table.put(0x1FCA, intArrayOf(0x0397, 0x0300)) + table.put(0xFA4B, intArrayOf(0x7891)) + table.put(0xFA4A, intArrayOf(0x7422)) + table.put(0x1FC9, intArrayOf(0x0395, 0x0301)) + table.put(0x1FC8, intArrayOf(0x0395, 0x0300)) + table.put(0xFA49, intArrayOf(0x722B)) + table.put(0x1FC7, intArrayOf(0x03B7, 0x0342, 0x0345)) + table.put(0xFA48, intArrayOf(0x716E)) + table.put(0x1FC6, intArrayOf(0x03B7, 0x0342)) + table.put(0xFA47, intArrayOf(0x6F22)) + table.put(0xFA46, intArrayOf(0x6E1A)) + table.put(0x1FC4, intArrayOf(0x03B7, 0x0301, 0x0345)) + table.put(0xFA45, intArrayOf(0x6D77)) + table.put(0x1FC3, intArrayOf(0x03B7, 0x0345)) + table.put(0xFA44, intArrayOf(0x6885)) + table.put(0x1FC2, intArrayOf(0x03B7, 0x0300, 0x0345)) + table.put(0xFA43, intArrayOf(0x6691)) + table.put(0x1FC1, intArrayOf(0x00A8, 0x0342)) + table.put(0xFA42, intArrayOf(0x65E2)) + table.put(0xFA41, intArrayOf(0x654F)) + table.put(0xFA40, intArrayOf(0x61F2)) + table.put(0x1FBE, intArrayOf(0x03B9)) + table.put(0xFA3F, intArrayOf(0x618E)) + table.put(0xFA3E, intArrayOf(0x6168)) + table.put(0x1FBC, intArrayOf(0x0391, 0x0345)) + table.put(0xFA3D, intArrayOf(0x6094)) + table.put(0x1FBB, intArrayOf(0x0391, 0x0301)) + table.put(0xFA3C, intArrayOf(0x5C6E)) + table.put(0x1FBA, intArrayOf(0x0391, 0x0300)) + table.put(0xFA3B, intArrayOf(0x5C64)) + table.put(0xFA3A, intArrayOf(0x58A8)) + table.put(0x1FB9, intArrayOf(0x0391, 0x0304)) + table.put(0x1FB8, intArrayOf(0x0391, 0x0306)) + table.put(0xFA39, intArrayOf(0x5840)) + table.put(0x1FB7, intArrayOf(0x03B1, 0x0342, 0x0345)) + table.put(0xFA38, intArrayOf(0x5668)) + table.put(0x1FB6, intArrayOf(0x03B1, 0x0342)) + table.put(0xFA37, intArrayOf(0x5606)) + table.put(0xFA36, intArrayOf(0x559D)) + table.put(0x1FB4, intArrayOf(0x03B1, 0x0301, 0x0345)) + table.put(0xFA35, intArrayOf(0x5351)) + table.put(0x1FB3, intArrayOf(0x03B1, 0x0345)) + table.put(0xFA34, intArrayOf(0x52E4)) + table.put(0x1FB2, intArrayOf(0x03B1, 0x0300, 0x0345)) + table.put(0xFA33, intArrayOf(0x52C9)) + table.put(0x1FB1, intArrayOf(0x03B1, 0x0304)) + table.put(0xFA32, intArrayOf(0x514D)) + table.put(0x1FB0, intArrayOf(0x03B1, 0x0306)) + table.put(0xFA31, intArrayOf(0x50E7)) + table.put(0xFA30, intArrayOf(0x4FAE)) + table.put(0x1FAF, intArrayOf(0x03A9, 0x0314, 0x0342, 0x0345)) + table.put(0x1FAE, intArrayOf(0x03A9, 0x0313, 0x0342, 0x0345)) + table.put(0x1FAD, intArrayOf(0x03A9, 0x0314, 0x0301, 0x0345)) + table.put(0x1FAC, intArrayOf(0x03A9, 0x0313, 0x0301, 0x0345)) + table.put(0xFA2D, intArrayOf(0x9DB4)) + table.put(0x1FAB, intArrayOf(0x03A9, 0x0314, 0x0300, 0x0345)) + table.put(0xFA2C, intArrayOf(0x9928)) + table.put(0x1FAA, intArrayOf(0x03A9, 0x0313, 0x0300, 0x0345)) + table.put(0xFA2B, intArrayOf(0x98FC)) + table.put(0xFA2A, intArrayOf(0x98EF)) + table.put(0x1FA9, intArrayOf(0x03A9, 0x0314, 0x0345)) + table.put(0x1FA8, intArrayOf(0x03A9, 0x0313, 0x0345)) + table.put(0x1FA7, intArrayOf(0x03C9, 0x0314, 0x0342, 0x0345)) + table.put(0x1FA6, intArrayOf(0x03C9, 0x0313, 0x0342, 0x0345)) + table.put(0x1FA5, intArrayOf(0x03C9, 0x0314, 0x0301, 0x0345)) + table.put(0xFA26, intArrayOf(0x90FD)) + table.put(0x1FA4, intArrayOf(0x03C9, 0x0313, 0x0301, 0x0345)) + table.put(0xFA25, intArrayOf(0x9038)) + table.put(0x1FA3, intArrayOf(0x03C9, 0x0314, 0x0300, 0x0345)) + table.put(0x1FA2, intArrayOf(0x03C9, 0x0313, 0x0300, 0x0345)) + table.put(0x1FA1, intArrayOf(0x03C9, 0x0314, 0x0345)) + table.put(0xFA22, intArrayOf(0x8AF8)) + table.put(0x1FA0, intArrayOf(0x03C9, 0x0313, 0x0345)) + table.put(0xFA20, intArrayOf(0x8612)) + table.put(0xFA1E, intArrayOf(0x7FBD)) + table.put(0xFA1D, intArrayOf(0x7CBE)) + table.put(0xFA1C, intArrayOf(0x9756)) + table.put(0xFA1B, intArrayOf(0x798F)) + table.put(0xFA1A, intArrayOf(0x7965)) + table.put(0xFA19, intArrayOf(0x795E)) + table.put(0xFA18, intArrayOf(0x793C)) + table.put(0xFA17, intArrayOf(0x76CA)) + table.put(0xFA16, intArrayOf(0x732A)) + table.put(0xFA15, intArrayOf(0x51DE)) + table.put(0xFA12, intArrayOf(0x6674)) + table.put(0xFA10, intArrayOf(0x585A)) + table.put(0xFA0D, intArrayOf(0x55C0)) + table.put(0xFA0C, intArrayOf(0x5140)) + table.put(0xFA0B, intArrayOf(0x5ED3)) + table.put(0xFA0A, intArrayOf(0x898B)) + table.put(0x212B, intArrayOf(0x0041, 0x030A)) + table.put(0x212A, intArrayOf(0x004B)) + table.put(0xFA09, intArrayOf(0x964D)) + table.put(0xFA08, intArrayOf(0x884C)) + table.put(0xFA07, intArrayOf(0x8F3B)) + table.put(0xFA06, intArrayOf(0x66B4)) + table.put(0xFA05, intArrayOf(0x6D1E)) + table.put(0xFA04, intArrayOf(0x5B85)) + table.put(0xFA03, intArrayOf(0x7CD6)) + table.put(0xFA02, intArrayOf(0x62D3)) + table.put(0x2126, intArrayOf(0x03A9)) + table.put(0xFA01, intArrayOf(0x5EA6)) + table.put(0xFA00, intArrayOf(0x5207)) + table.put(0x0B94, intArrayOf(0x0B92, 0x0BD7)) + table.put(0x1026, intArrayOf(0x1025, 0x102E)) + table.put(0x0B5D, intArrayOf(0x0B22, 0x0B3C)) + table.put(0x0B5C, intArrayOf(0x0B21, 0x0B3C)) + table.put(0x1F9F, intArrayOf(0x0397, 0x0314, 0x0342, 0x0345)) + table.put(0x1F9E, intArrayOf(0x0397, 0x0313, 0x0342, 0x0345)) + table.put(0x1F9D, intArrayOf(0x0397, 0x0314, 0x0301, 0x0345)) + table.put(0x1F9C, intArrayOf(0x0397, 0x0313, 0x0301, 0x0345)) + table.put(0x1F9B, intArrayOf(0x0397, 0x0314, 0x0300, 0x0345)) + table.put(0x1F9A, intArrayOf(0x0397, 0x0313, 0x0300, 0x0345)) + table.put(0x0B4C, intArrayOf(0x0B47, 0x0B57)) + table.put(0x0B4B, intArrayOf(0x0B47, 0x0B3E)) + table.put(0x1F99, intArrayOf(0x0397, 0x0314, 0x0345)) + table.put(0x1F98, intArrayOf(0x0397, 0x0313, 0x0345)) + table.put(0x1F97, intArrayOf(0x03B7, 0x0314, 0x0342, 0x0345)) + table.put(0x1F96, intArrayOf(0x03B7, 0x0313, 0x0342, 0x0345)) + table.put(0x1F95, intArrayOf(0x03B7, 0x0314, 0x0301, 0x0345)) + table.put(0x1F94, intArrayOf(0x03B7, 0x0313, 0x0301, 0x0345)) + table.put(0x1F93, intArrayOf(0x03B7, 0x0314, 0x0300, 0x0345)) + table.put(0x1F92, intArrayOf(0x03B7, 0x0313, 0x0300, 0x0345)) + table.put(0x1F91, intArrayOf(0x03B7, 0x0314, 0x0345)) + table.put(0x1F90, intArrayOf(0x03B7, 0x0313, 0x0345)) + table.put(0x0B48, intArrayOf(0x0B47, 0x0B56)) + table.put(0x1F8F, intArrayOf(0x0391, 0x0314, 0x0342, 0x0345)) + table.put(0x1F8E, intArrayOf(0x0391, 0x0313, 0x0342, 0x0345)) + table.put(0x1F8D, intArrayOf(0x0391, 0x0314, 0x0301, 0x0345)) + table.put(0x1F8C, intArrayOf(0x0391, 0x0313, 0x0301, 0x0345)) + table.put(0x1F8B, intArrayOf(0x0391, 0x0314, 0x0300, 0x0345)) + table.put(0x1F8A, intArrayOf(0x0391, 0x0313, 0x0300, 0x0345)) + table.put(0x1F89, intArrayOf(0x0391, 0x0314, 0x0345)) + table.put(0x1F88, intArrayOf(0x0391, 0x0313, 0x0345)) + table.put(0x1F87, intArrayOf(0x03B1, 0x0314, 0x0342, 0x0345)) + table.put(0x1F86, intArrayOf(0x03B1, 0x0313, 0x0342, 0x0345)) + table.put(0x1F85, intArrayOf(0x03B1, 0x0314, 0x0301, 0x0345)) + table.put(0x1F84, intArrayOf(0x03B1, 0x0313, 0x0301, 0x0345)) + table.put(0x1F83, intArrayOf(0x03B1, 0x0314, 0x0300, 0x0345)) + table.put(0x1F82, intArrayOf(0x03B1, 0x0313, 0x0300, 0x0345)) + table.put(0x1F81, intArrayOf(0x03B1, 0x0314, 0x0345)) + table.put(0x1F80, intArrayOf(0x03B1, 0x0313, 0x0345)) + table.put(0x1F7D, intArrayOf(0x03C9, 0x0301)) + table.put(0x1F7C, intArrayOf(0x03C9, 0x0300)) + table.put(0x1F7B, intArrayOf(0x03C5, 0x0301)) + table.put(0x1F7A, intArrayOf(0x03C5, 0x0300)) + table.put(0x1F79, intArrayOf(0x03BF, 0x0301)) + table.put(0x1F78, intArrayOf(0x03BF, 0x0300)) + table.put(0x1F77, intArrayOf(0x03B9, 0x0301)) + table.put(0x1F76, intArrayOf(0x03B9, 0x0300)) + table.put(0x1F75, intArrayOf(0x03B7, 0x0301)) + table.put(0x0233, intArrayOf(0x0079, 0x0304)) + table.put(0x1F74, intArrayOf(0x03B7, 0x0300)) + table.put(0x0232, intArrayOf(0x0059, 0x0304)) + table.put(0x1F73, intArrayOf(0x03B5, 0x0301)) + table.put(0x0231, intArrayOf(0x006F, 0x0307, 0x0304)) + table.put(0x1F72, intArrayOf(0x03B5, 0x0300)) + table.put(0x0230, intArrayOf(0x004F, 0x0307, 0x0304)) + table.put(0x1F71, intArrayOf(0x03B1, 0x0301)) + table.put(0x1F70, intArrayOf(0x03B1, 0x0300)) + table.put(0x022F, intArrayOf(0x006F, 0x0307)) + table.put(0x022E, intArrayOf(0x004F, 0x0307)) + table.put(0x1F6F, intArrayOf(0x03A9, 0x0314, 0x0342)) + table.put(0x022D, intArrayOf(0x006F, 0x0303, 0x0304)) + table.put(0x1F6E, intArrayOf(0x03A9, 0x0313, 0x0342)) + table.put(0x022C, intArrayOf(0x004F, 0x0303, 0x0304)) + table.put(0x1F6D, intArrayOf(0x03A9, 0x0314, 0x0301)) + table.put(0x022B, intArrayOf(0x006F, 0x0308, 0x0304)) + table.put(0x1F6C, intArrayOf(0x03A9, 0x0313, 0x0301)) + table.put(0x022A, intArrayOf(0x004F, 0x0308, 0x0304)) + table.put(0x1F6B, intArrayOf(0x03A9, 0x0314, 0x0300)) + table.put(0x1F6A, intArrayOf(0x03A9, 0x0313, 0x0300)) + table.put(0x0229, intArrayOf(0x0065, 0x0327)) + table.put(0x0228, intArrayOf(0x0045, 0x0327)) + table.put(0x1F69, intArrayOf(0x03A9, 0x0314)) + table.put(0x0227, intArrayOf(0x0061, 0x0307)) + table.put(0x1F68, intArrayOf(0x03A9, 0x0313)) + table.put(0x0226, intArrayOf(0x0041, 0x0307)) + table.put(0x1F67, intArrayOf(0x03C9, 0x0314, 0x0342)) + table.put(0x1F66, intArrayOf(0x03C9, 0x0313, 0x0342)) + table.put(0x1F65, intArrayOf(0x03C9, 0x0314, 0x0301)) + table.put(0x1F64, intArrayOf(0x03C9, 0x0313, 0x0301)) + table.put(0x1F63, intArrayOf(0x03C9, 0x0314, 0x0300)) + table.put(0x1F62, intArrayOf(0x03C9, 0x0313, 0x0300)) + table.put(0x1F61, intArrayOf(0x03C9, 0x0314)) + table.put(0x1F60, intArrayOf(0x03C9, 0x0313)) + table.put(0x021F, intArrayOf(0x0068, 0x030C)) + table.put(0x021E, intArrayOf(0x0048, 0x030C)) + table.put(0x1F5F, intArrayOf(0x03A5, 0x0314, 0x0342)) + table.put(0x1F5D, intArrayOf(0x03A5, 0x0314, 0x0301)) + table.put(0x021B, intArrayOf(0x0074, 0x0326)) + table.put(0x021A, intArrayOf(0x0054, 0x0326)) + table.put(0x1F5B, intArrayOf(0x03A5, 0x0314, 0x0300)) + table.put(0x0219, intArrayOf(0x0073, 0x0326)) + table.put(0x0218, intArrayOf(0x0053, 0x0326)) + table.put(0x1F59, intArrayOf(0x03A5, 0x0314)) + table.put(0x0217, intArrayOf(0x0075, 0x0311)) + table.put(0x0216, intArrayOf(0x0055, 0x0311)) + table.put(0x1F57, intArrayOf(0x03C5, 0x0314, 0x0342)) + table.put(0x0215, intArrayOf(0x0075, 0x030F)) + table.put(0x1F56, intArrayOf(0x03C5, 0x0313, 0x0342)) + table.put(0x0214, intArrayOf(0x0055, 0x030F)) + table.put(0x1F55, intArrayOf(0x03C5, 0x0314, 0x0301)) + table.put(0x0213, intArrayOf(0x0072, 0x0311)) + table.put(0x1F54, intArrayOf(0x03C5, 0x0313, 0x0301)) + table.put(0x0212, intArrayOf(0x0052, 0x0311)) + table.put(0x1F53, intArrayOf(0x03C5, 0x0314, 0x0300)) + table.put(0x0211, intArrayOf(0x0072, 0x030F)) + table.put(0x1F52, intArrayOf(0x03C5, 0x0313, 0x0300)) + table.put(0x0210, intArrayOf(0x0052, 0x030F)) + table.put(0x1F51, intArrayOf(0x03C5, 0x0314)) + table.put(0x1F50, intArrayOf(0x03C5, 0x0313)) + table.put(0x020F, intArrayOf(0x006F, 0x0311)) + table.put(0x020E, intArrayOf(0x004F, 0x0311)) + table.put(0x020D, intArrayOf(0x006F, 0x030F)) + table.put(0x020C, intArrayOf(0x004F, 0x030F)) + table.put(0x1F4D, intArrayOf(0x039F, 0x0314, 0x0301)) + table.put(0x020B, intArrayOf(0x0069, 0x0311)) + table.put(0x1F4C, intArrayOf(0x039F, 0x0313, 0x0301)) + table.put(0x020A, intArrayOf(0x0049, 0x0311)) + table.put(0x1F4B, intArrayOf(0x039F, 0x0314, 0x0300)) + table.put(0x1F4A, intArrayOf(0x039F, 0x0313, 0x0300)) + table.put(0x0209, intArrayOf(0x0069, 0x030F)) + table.put(0x0208, intArrayOf(0x0049, 0x030F)) + table.put(0x1F49, intArrayOf(0x039F, 0x0314)) + table.put(0x0207, intArrayOf(0x0065, 0x0311)) + table.put(0x1F48, intArrayOf(0x039F, 0x0313)) + table.put(0x0206, intArrayOf(0x0045, 0x0311)) + table.put(0x0205, intArrayOf(0x0065, 0x030F)) + table.put(0x0204, intArrayOf(0x0045, 0x030F)) + table.put(0x1F45, intArrayOf(0x03BF, 0x0314, 0x0301)) + table.put(0x0203, intArrayOf(0x0061, 0x0311)) + table.put(0x1F44, intArrayOf(0x03BF, 0x0313, 0x0301)) + table.put(0x0202, intArrayOf(0x0041, 0x0311)) + table.put(0x1F43, intArrayOf(0x03BF, 0x0314, 0x0300)) + table.put(0x0201, intArrayOf(0x0061, 0x030F)) + table.put(0x1F42, intArrayOf(0x03BF, 0x0313, 0x0300)) + table.put(0x0200, intArrayOf(0x0041, 0x030F)) + table.put(0x1F41, intArrayOf(0x03BF, 0x0314)) + table.put(0x1F40, intArrayOf(0x03BF, 0x0313)) + table.put(0x2F9FF, intArrayOf(0x980B)) + table.put(0x2F9FE, intArrayOf(0x980B)) + table.put(0x1F3F, intArrayOf(0x0399, 0x0314, 0x0342)) + table.put(0x2F9FD, intArrayOf(0x29496)) + table.put(0x1F3E, intArrayOf(0x0399, 0x0313, 0x0342)) + table.put(0x2F9FC, intArrayOf(0x4AB2)) + table.put(0x1F3D, intArrayOf(0x0399, 0x0314, 0x0301)) + table.put(0x2F9FB, intArrayOf(0x2940A)) + table.put(0x1F3C, intArrayOf(0x0399, 0x0313, 0x0301)) + table.put(0x2F9FA, intArrayOf(0x97E0)) + table.put(0x1F3B, intArrayOf(0x0399, 0x0314, 0x0300)) + table.put(0x1F3A, intArrayOf(0x0399, 0x0313, 0x0300)) + table.put(0x2F9F9, intArrayOf(0x4A76)) + table.put(0x2F9F8, intArrayOf(0x4A6E)) + table.put(0x1F39, intArrayOf(0x0399, 0x0314)) + table.put(0x2F9F7, intArrayOf(0x2921A)) + table.put(0x1F38, intArrayOf(0x0399, 0x0313)) + table.put(0x2F9F6, intArrayOf(0x29145)) + table.put(0x1F37, intArrayOf(0x03B9, 0x0314, 0x0342)) + table.put(0x2F9F5, intArrayOf(0x9723)) + table.put(0x1F36, intArrayOf(0x03B9, 0x0313, 0x0342)) + table.put(0x2F9F4, intArrayOf(0x5DB2)) + table.put(0x1F35, intArrayOf(0x03B9, 0x0314, 0x0301)) + table.put(0x2F9F3, intArrayOf(0x96C3)) + table.put(0x1F34, intArrayOf(0x03B9, 0x0313, 0x0301)) + table.put(0x2F9F2, intArrayOf(0x49E6)) + table.put(0x1F33, intArrayOf(0x03B9, 0x0314, 0x0300)) + table.put(0x2F9F1, intArrayOf(0x28D77)) + table.put(0x1F32, intArrayOf(0x03B9, 0x0313, 0x0300)) + table.put(0x2F9F0, intArrayOf(0x95B7)) + table.put(0x1F31, intArrayOf(0x03B9, 0x0314)) + table.put(0x1F30, intArrayOf(0x03B9, 0x0313)) + table.put(0x2F9EF, intArrayOf(0x4995)) + table.put(0x2F9EE, intArrayOf(0x958B)) + table.put(0x1F2F, intArrayOf(0x0397, 0x0314, 0x0342)) + table.put(0x2F9ED, intArrayOf(0x28BFA)) + table.put(0x1F2E, intArrayOf(0x0397, 0x0313, 0x0342)) + table.put(0x2F9EC, intArrayOf(0x9415)) + table.put(0x1F2D, intArrayOf(0x0397, 0x0314, 0x0301)) + table.put(0x2F9EB, intArrayOf(0x93F9)) + table.put(0x1F2C, intArrayOf(0x0397, 0x0313, 0x0301)) + table.put(0x2F9EA, intArrayOf(0x927C)) + table.put(0x1F2B, intArrayOf(0x0397, 0x0314, 0x0300)) + table.put(0x1F2A, intArrayOf(0x0397, 0x0313, 0x0300)) + table.put(0x2F9E9, intArrayOf(0x92D8)) + table.put(0x2F9E8, intArrayOf(0x92D7)) + table.put(0x1F29, intArrayOf(0x0397, 0x0314)) + table.put(0x2F9E7, intArrayOf(0x9238)) + table.put(0x1F28, intArrayOf(0x0397, 0x0313)) + table.put(0x2F9E6, intArrayOf(0x911B)) + table.put(0x1F27, intArrayOf(0x03B7, 0x0314, 0x0342)) + table.put(0x2F9E5, intArrayOf(0x2872E)) + table.put(0x1F26, intArrayOf(0x03B7, 0x0313, 0x0342)) + table.put(0x2F9E4, intArrayOf(0x9111)) + table.put(0x1F25, intArrayOf(0x03B7, 0x0314, 0x0301)) + table.put(0x2F9E3, intArrayOf(0x90F1)) + table.put(0x1F24, intArrayOf(0x03B7, 0x0313, 0x0301)) + table.put(0x2F9E2, intArrayOf(0x9094)) + table.put(0x1F23, intArrayOf(0x03B7, 0x0314, 0x0300)) + table.put(0x2F9E1, intArrayOf(0x285ED)) + table.put(0x1F22, intArrayOf(0x03B7, 0x0313, 0x0300)) + table.put(0x2F9E0, intArrayOf(0x285D2)) + table.put(0x1F21, intArrayOf(0x03B7, 0x0314)) + table.put(0x1F20, intArrayOf(0x03B7, 0x0313)) + table.put(0x2F9DF, intArrayOf(0x8F38)) + table.put(0x2F9DE, intArrayOf(0x8ED4)) + table.put(0x2F9DD, intArrayOf(0x208DE)) + table.put(0x2F9DC, intArrayOf(0x8DF0)) + table.put(0x1F1D, intArrayOf(0x0395, 0x0314, 0x0301)) + table.put(0x2F9DB, intArrayOf(0x8DBC)) + table.put(0x1F1C, intArrayOf(0x0395, 0x0313, 0x0301)) + table.put(0x2F9DA, intArrayOf(0x8DCB)) + table.put(0x1F1B, intArrayOf(0x0395, 0x0314, 0x0300)) + table.put(0x1F1A, intArrayOf(0x0395, 0x0313, 0x0300)) + table.put(0x2F9D9, intArrayOf(0x20804)) + table.put(0x2F9D8, intArrayOf(0x27F2F)) + table.put(0x1F19, intArrayOf(0x0395, 0x0314)) + table.put(0x2F9D7, intArrayOf(0x8D77)) + table.put(0x1F18, intArrayOf(0x0395, 0x0313)) + table.put(0x2F9D6, intArrayOf(0x8D1B)) + table.put(0x2F9D5, intArrayOf(0x8CC1)) + table.put(0x2F9D4, intArrayOf(0x8CAB)) + table.put(0x1F15, intArrayOf(0x03B5, 0x0314, 0x0301)) + table.put(0x2F9D3, intArrayOf(0x27CA8)) + table.put(0x1F14, intArrayOf(0x03B5, 0x0313, 0x0301)) + table.put(0x2F9D2, intArrayOf(0x8C55)) + table.put(0x1F13, intArrayOf(0x03B5, 0x0314, 0x0300)) + table.put(0x2F9D1, intArrayOf(0x8B8A)) + table.put(0x1F12, intArrayOf(0x03B5, 0x0313, 0x0300)) + table.put(0x2F9D0, intArrayOf(0x8AED)) + table.put(0x1F11, intArrayOf(0x03B5, 0x0314)) + table.put(0x1F10, intArrayOf(0x03B5, 0x0313)) + table.put(0x2F9CF, intArrayOf(0x8AA0)) + table.put(0x2F9CE, intArrayOf(0x46C7)) + table.put(0x1F0F, intArrayOf(0x0391, 0x0314, 0x0342)) + table.put(0x2F9CD, intArrayOf(0x46BE)) + table.put(0x1F0E, intArrayOf(0x0391, 0x0313, 0x0342)) + table.put(0x2F9CC, intArrayOf(0x27966)) + table.put(0x1F0D, intArrayOf(0x0391, 0x0314, 0x0301)) + table.put(0x2F9CB, intArrayOf(0x278AE)) + table.put(0x1F0C, intArrayOf(0x0391, 0x0313, 0x0301)) + table.put(0x2F9CA, intArrayOf(0x34BB)) + table.put(0x1F0B, intArrayOf(0x0391, 0x0314, 0x0300)) + table.put(0x1F0A, intArrayOf(0x0391, 0x0313, 0x0300)) + table.put(0x2F9C9, intArrayOf(0x88FA)) + table.put(0x2F9C8, intArrayOf(0x4635)) + table.put(0x1F09, intArrayOf(0x0391, 0x0314)) + table.put(0x2F9C7, intArrayOf(0x88DE)) + table.put(0x1F08, intArrayOf(0x0391, 0x0313)) + table.put(0x2F9C6, intArrayOf(0x88D7)) + table.put(0x1F07, intArrayOf(0x03B1, 0x0314, 0x0342)) + table.put(0x2F9C5, intArrayOf(0x27667)) + table.put(0x1F06, intArrayOf(0x03B1, 0x0313, 0x0342)) + table.put(0x2F9C4, intArrayOf(0x8863)) + table.put(0x1F05, intArrayOf(0x03B1, 0x0314, 0x0301)) + table.put(0x2F9C3, intArrayOf(0x8860)) + table.put(0x1F04, intArrayOf(0x03B1, 0x0313, 0x0301)) + table.put(0x2F9C2, intArrayOf(0x45F9)) + table.put(0x1F03, intArrayOf(0x03B1, 0x0314, 0x0300)) + table.put(0x2F9C1, intArrayOf(0x8801)) + table.put(0x1F02, intArrayOf(0x03B1, 0x0313, 0x0300)) + table.put(0x2F9C0, intArrayOf(0x87E1)) + table.put(0x1F01, intArrayOf(0x03B1, 0x0314)) + table.put(0x1F00, intArrayOf(0x03B1, 0x0313)) + table.put(0x2F9BF, intArrayOf(0x45D7)) + table.put(0x2F9BE, intArrayOf(0x8786)) + table.put(0x2F9BD, intArrayOf(0x876B)) + table.put(0x2F9BC, intArrayOf(0x8728)) + table.put(0x2F9BB, intArrayOf(0x8779)) + table.put(0x2F9BA, intArrayOf(0x86E2)) + table.put(0x2F9B9, intArrayOf(0x870E)) + table.put(0x2F9B8, intArrayOf(0x8688)) + table.put(0x2F9B7, intArrayOf(0x86A9)) + table.put(0x2F9B6, intArrayOf(0x8669)) + table.put(0x2F9B5, intArrayOf(0x8667)) + table.put(0x2F9B4, intArrayOf(0x865C)) + table.put(0x2F9B3, intArrayOf(0x8650)) + table.put(0x2F9B2, intArrayOf(0x456B)) + table.put(0x2F9B1, intArrayOf(0x270D2)) + table.put(0x2F9B0, intArrayOf(0x26FB1)) + table.put(0x2F9AF, intArrayOf(0x4561)) + table.put(0x2F9AE, intArrayOf(0x455D)) + table.put(0x2F9AD, intArrayOf(0x26F2C)) + table.put(0x2F9AC, intArrayOf(0x8564)) + table.put(0x2F9AB, intArrayOf(0x273CA)) + table.put(0x2F9AA, intArrayOf(0x8516)) + table.put(0x2F9A9, intArrayOf(0x84F3)) + table.put(0x2F9A8, intArrayOf(0x84F1)) + table.put(0x2F9A7, intArrayOf(0x452B)) + table.put(0x2F9A6, intArrayOf(0x26CD5)) + table.put(0x2F9A5, intArrayOf(0x26D6B)) + table.put(0x2F9A4, intArrayOf(0x26C36)) + table.put(0x2F9A3, intArrayOf(0x83DC)) + table.put(0x2F9A2, intArrayOf(0x83CC)) + table.put(0x2F9A1, intArrayOf(0x83CA)) + table.put(0x2F9A0, intArrayOf(0x8353)) + table.put(0x01FF, intArrayOf(0x00F8, 0x0301)) + table.put(0x01FE, intArrayOf(0x00D8, 0x0301)) + table.put(0x01FD, intArrayOf(0x00E6, 0x0301)) + table.put(0x01FC, intArrayOf(0x00C6, 0x0301)) + table.put(0x01FB, intArrayOf(0x0061, 0x030A, 0x0301)) + table.put(0x01FA, intArrayOf(0x0041, 0x030A, 0x0301)) + table.put(0x01F9, intArrayOf(0x006E, 0x0300)) + table.put(0x01F8, intArrayOf(0x004E, 0x0300)) + table.put(0x01F5, intArrayOf(0x0067, 0x0301)) + table.put(0x01F4, intArrayOf(0x0047, 0x0301)) + table.put(0x01F0, intArrayOf(0x006A, 0x030C)) + table.put(0x01EF, intArrayOf(0x0292, 0x030C)) + table.put(0x01EE, intArrayOf(0x01B7, 0x030C)) + table.put(0x01ED, intArrayOf(0x006F, 0x0328, 0x0304)) + table.put(0x01EC, intArrayOf(0x004F, 0x0328, 0x0304)) + table.put(0x01EB, intArrayOf(0x006F, 0x0328)) + table.put(0x01EA, intArrayOf(0x004F, 0x0328)) + table.put(0x01E9, intArrayOf(0x006B, 0x030C)) + table.put(0x01E8, intArrayOf(0x004B, 0x030C)) + table.put(0x01E7, intArrayOf(0x0067, 0x030C)) + table.put(0x01E6, intArrayOf(0x0047, 0x030C)) + table.put(0x01E3, intArrayOf(0x00E6, 0x0304)) + table.put(0x01E2, intArrayOf(0x00C6, 0x0304)) + table.put(0x01E1, intArrayOf(0x0061, 0x0307, 0x0304)) + table.put(0x01E0, intArrayOf(0x0041, 0x0307, 0x0304)) + table.put(0x01DF, intArrayOf(0x0061, 0x0308, 0x0304)) + table.put(0x01DE, intArrayOf(0x0041, 0x0308, 0x0304)) + table.put(0x01DC, intArrayOf(0x0075, 0x0308, 0x0300)) + table.put(0x01DB, intArrayOf(0x0055, 0x0308, 0x0300)) + table.put(0x01DA, intArrayOf(0x0075, 0x0308, 0x030C)) + table.put(0x01D9, intArrayOf(0x0055, 0x0308, 0x030C)) + table.put(0x01D8, intArrayOf(0x0075, 0x0308, 0x0301)) + table.put(0x01D7, intArrayOf(0x0055, 0x0308, 0x0301)) + table.put(0x01D6, intArrayOf(0x0075, 0x0308, 0x0304)) + table.put(0x01D5, intArrayOf(0x0055, 0x0308, 0x0304)) + table.put(0x01D4, intArrayOf(0x0075, 0x030C)) + table.put(0x01D3, intArrayOf(0x0055, 0x030C)) + table.put(0x01D2, intArrayOf(0x006F, 0x030C)) + table.put(0x01D1, intArrayOf(0x004F, 0x030C)) + table.put(0x01D0, intArrayOf(0x0069, 0x030C)) + table.put(0x01CF, intArrayOf(0x0049, 0x030C)) + table.put(0x01CE, intArrayOf(0x0061, 0x030C)) + table.put(0x01CD, intArrayOf(0x0041, 0x030C)) + table.put(0x1EF9, intArrayOf(0x0079, 0x0303)) + table.put(0x1EF8, intArrayOf(0x0059, 0x0303)) + table.put(0x1EF7, intArrayOf(0x0079, 0x0309)) + table.put(0x1EF6, intArrayOf(0x0059, 0x0309)) + table.put(0x1EF5, intArrayOf(0x0079, 0x0323)) + table.put(0x1EF4, intArrayOf(0x0059, 0x0323)) + table.put(0x1EF3, intArrayOf(0x0079, 0x0300)) + table.put(0x1EF2, intArrayOf(0x0059, 0x0300)) + table.put(0x01B0, intArrayOf(0x0075, 0x031B)) + table.put(0x1EF1, intArrayOf(0x0075, 0x031B, 0x0323)) + table.put(0x1EF0, intArrayOf(0x0055, 0x031B, 0x0323)) + table.put(0x2F99F, intArrayOf(0x8457)) + table.put(0x01AF, intArrayOf(0x0055, 0x031B)) + table.put(0x2F99E, intArrayOf(0x83E7)) + table.put(0x1EEF, intArrayOf(0x0075, 0x031B, 0x0303)) + table.put(0x2F99D, intArrayOf(0x83BD)) + table.put(0x1EEE, intArrayOf(0x0055, 0x031B, 0x0303)) + table.put(0x2F99C, intArrayOf(0x8323)) + table.put(0x1EED, intArrayOf(0x0075, 0x031B, 0x0309)) + table.put(0x2F99B, intArrayOf(0x83AD)) + table.put(0x1EEC, intArrayOf(0x0055, 0x031B, 0x0309)) + table.put(0x2F99A, intArrayOf(0x8363)) + table.put(0x1EEB, intArrayOf(0x0075, 0x031B, 0x0300)) + table.put(0x1EEA, intArrayOf(0x0055, 0x031B, 0x0300)) + table.put(0x2F999, intArrayOf(0x831D)) + table.put(0x2F998, intArrayOf(0x82E5)) + table.put(0x1EE9, intArrayOf(0x0075, 0x031B, 0x0301)) + table.put(0x2F997, intArrayOf(0x26B3C)) + table.put(0x1EE8, intArrayOf(0x0055, 0x031B, 0x0301)) + table.put(0x2F996, intArrayOf(0x82E6)) + table.put(0x1EE7, intArrayOf(0x0075, 0x0309)) + table.put(0x2F995, intArrayOf(0x82BD)) + table.put(0x1EE6, intArrayOf(0x0055, 0x0309)) + table.put(0x2F994, intArrayOf(0x82B3)) + table.put(0x1EE5, intArrayOf(0x0075, 0x0323)) + table.put(0x2F993, intArrayOf(0x82B1)) + table.put(0x1EE4, intArrayOf(0x0055, 0x0323)) + table.put(0x2F992, intArrayOf(0x52B3)) + table.put(0x1EE3, intArrayOf(0x006F, 0x031B, 0x0323)) + table.put(0x2F991, intArrayOf(0x829D)) + table.put(0x01A1, intArrayOf(0x006F, 0x031B)) + table.put(0x1EE2, intArrayOf(0x004F, 0x031B, 0x0323)) + table.put(0x2F990, intArrayOf(0x828B)) + table.put(0x01A0, intArrayOf(0x004F, 0x031B)) + table.put(0x1EE1, intArrayOf(0x006F, 0x031B, 0x0303)) + table.put(0x1EE0, intArrayOf(0x004F, 0x031B, 0x0303)) + table.put(0x2F98F, intArrayOf(0x8291)) + table.put(0x2F98E, intArrayOf(0x446B)) + table.put(0x1EDF, intArrayOf(0x006F, 0x031B, 0x0309)) + table.put(0x2F98D, intArrayOf(0x8F9E)) + table.put(0x1EDE, intArrayOf(0x004F, 0x031B, 0x0309)) + table.put(0x2F98C, intArrayOf(0x8204)) + table.put(0x1EDD, intArrayOf(0x006F, 0x031B, 0x0300)) + table.put(0x2F98B, intArrayOf(0x8201)) + table.put(0x1EDC, intArrayOf(0x004F, 0x031B, 0x0300)) + table.put(0x2F98A, intArrayOf(0x2339C)) + table.put(0x1EDB, intArrayOf(0x006F, 0x031B, 0x0301)) + table.put(0x1EDA, intArrayOf(0x004F, 0x031B, 0x0301)) + table.put(0x2F989, intArrayOf(0x23393)) + table.put(0x2F988, intArrayOf(0x267B5)) + table.put(0x1ED9, intArrayOf(0x006F, 0x0323, 0x0302)) + table.put(0x2F987, intArrayOf(0x267A7)) + table.put(0x04F9, intArrayOf(0x044B, 0x0308)) + table.put(0x1ED8, intArrayOf(0x004F, 0x0323, 0x0302)) + table.put(0x2F986, intArrayOf(0x5AB5)) + table.put(0x04F8, intArrayOf(0x042B, 0x0308)) + table.put(0x1ED7, intArrayOf(0x006F, 0x0302, 0x0303)) + table.put(0x2F985, intArrayOf(0x813E)) + table.put(0x1ED6, intArrayOf(0x004F, 0x0302, 0x0303)) + table.put(0x2F984, intArrayOf(0x440B)) + table.put(0x1ED5, intArrayOf(0x006F, 0x0302, 0x0309)) + table.put(0x2F983, intArrayOf(0x8103)) + table.put(0x04F5, intArrayOf(0x0447, 0x0308)) + table.put(0x1ED4, intArrayOf(0x004F, 0x0302, 0x0309)) + table.put(0x2F982, intArrayOf(0x80B2)) + table.put(0x04F4, intArrayOf(0x0427, 0x0308)) + table.put(0x1ED3, intArrayOf(0x006F, 0x0302, 0x0300)) + table.put(0x2F981, intArrayOf(0x43D5)) + table.put(0x04F3, intArrayOf(0x0443, 0x030B)) + table.put(0x1ED2, intArrayOf(0x004F, 0x0302, 0x0300)) + table.put(0x2F980, intArrayOf(0x2335F)) + table.put(0x04F2, intArrayOf(0x0423, 0x030B)) + table.put(0x1ED1, intArrayOf(0x006F, 0x0302, 0x0301)) + table.put(0x04F1, intArrayOf(0x0443, 0x0308)) + table.put(0x1ED0, intArrayOf(0x004F, 0x0302, 0x0301)) + table.put(0x04F0, intArrayOf(0x0423, 0x0308)) + table.put(0x2F97F, intArrayOf(0x8070)) + table.put(0x2F97E, intArrayOf(0x265A8)) + table.put(0x1ECF, intArrayOf(0x006F, 0x0309)) + table.put(0x2F97D, intArrayOf(0x8060)) + table.put(0x04EF, intArrayOf(0x0443, 0x0304)) + table.put(0x1ECE, intArrayOf(0x004F, 0x0309)) + table.put(0x2F97C, intArrayOf(0x26523)) + table.put(0x04EE, intArrayOf(0x0423, 0x0304)) + table.put(0x1ECD, intArrayOf(0x006F, 0x0323)) + table.put(0x2F97B, intArrayOf(0x264DA)) + table.put(0x04ED, intArrayOf(0x044D, 0x0308)) + table.put(0x1ECC, intArrayOf(0x004F, 0x0323)) + table.put(0x2F97A, intArrayOf(0x8005)) + table.put(0x04EC, intArrayOf(0x042D, 0x0308)) + table.put(0x1ECB, intArrayOf(0x0069, 0x0323)) + table.put(0x04EB, intArrayOf(0x04E9, 0x0308)) + table.put(0x1ECA, intArrayOf(0x0049, 0x0323)) + table.put(0x04EA, intArrayOf(0x04E8, 0x0308)) + table.put(0x2F979, intArrayOf(0x7FFA)) + table.put(0x0DDE, intArrayOf(0x0DD9, 0x0DDF)) + table.put(0x2F978, intArrayOf(0x7F95)) + table.put(0x0DDD, intArrayOf(0x0DD9, 0x0DCF, 0x0DCA)) + table.put(0x1EC9, intArrayOf(0x0069, 0x0309)) + table.put(0x2F977, intArrayOf(0x2633E)) + table.put(0x0DDC, intArrayOf(0x0DD9, 0x0DCF)) + table.put(0x1EC8, intArrayOf(0x0049, 0x0309)) + table.put(0x2F976, intArrayOf(0x7F7A)) + table.put(0x1EC7, intArrayOf(0x0065, 0x0323, 0x0302)) + table.put(0x2F975, intArrayOf(0x262D9)) + table.put(0x04E7, intArrayOf(0x043E, 0x0308)) + table.put(0x0DDA, intArrayOf(0x0DD9, 0x0DCA)) + table.put(0x1EC6, intArrayOf(0x0045, 0x0323, 0x0302)) + table.put(0x2F974, intArrayOf(0x4359)) + table.put(0x04E6, intArrayOf(0x041E, 0x0308)) + table.put(0x1EC5, intArrayOf(0x0065, 0x0302, 0x0303)) + table.put(0x2F973, intArrayOf(0x26247)) + table.put(0x04E5, intArrayOf(0x0438, 0x0308)) + table.put(0x1EC4, intArrayOf(0x0045, 0x0302, 0x0303)) + table.put(0x2F972, intArrayOf(0x26228)) + table.put(0x04E4, intArrayOf(0x0418, 0x0308)) + table.put(0x1EC3, intArrayOf(0x0065, 0x0302, 0x0309)) + table.put(0x2F971, intArrayOf(0x4334)) + table.put(0x04E3, intArrayOf(0x0438, 0x0304)) + table.put(0x1EC2, intArrayOf(0x0045, 0x0302, 0x0309)) + table.put(0x2F970, intArrayOf(0x7E45)) + table.put(0x04E2, intArrayOf(0x0418, 0x0304)) + table.put(0x1EC1, intArrayOf(0x0065, 0x0302, 0x0300)) + table.put(0x1EC0, intArrayOf(0x0045, 0x0302, 0x0300)) + table.put(0xF9FF, intArrayOf(0x523A)) + table.put(0xF9FE, intArrayOf(0x8336)) + table.put(0x2F96F, intArrayOf(0x7E02)) + table.put(0xF9FD, intArrayOf(0x4EC0)) + table.put(0x2F96E, intArrayOf(0x7DC7)) + table.put(0x1EBF, intArrayOf(0x0065, 0x0302, 0x0301)) + table.put(0xF9FC, intArrayOf(0x8B58)) + table.put(0x2F96D, intArrayOf(0x4301)) + table.put(0x04DF, intArrayOf(0x0437, 0x0308)) + table.put(0x1EBE, intArrayOf(0x0045, 0x0302, 0x0301)) + table.put(0xF9FB, intArrayOf(0x7099)) + table.put(0x2F96C, intArrayOf(0x7D63)) + table.put(0x04DE, intArrayOf(0x0417, 0x0308)) + table.put(0x1EBD, intArrayOf(0x0065, 0x0303)) + table.put(0xF9FA, intArrayOf(0x72C0)) + table.put(0x2F96B, intArrayOf(0x25F86)) + table.put(0x04DD, intArrayOf(0x0436, 0x0308)) + table.put(0x1EBC, intArrayOf(0x0045, 0x0303)) + table.put(0x2F96A, intArrayOf(0x7D00)) + table.put(0x04DC, intArrayOf(0x0416, 0x0308)) + table.put(0x1EBB, intArrayOf(0x0065, 0x0309)) + table.put(0x04DB, intArrayOf(0x04D9, 0x0308)) + table.put(0x1EBA, intArrayOf(0x0045, 0x0309)) + table.put(0x04DA, intArrayOf(0x04D8, 0x0308)) + table.put(0xF9F9, intArrayOf(0x7C92)) + table.put(0xF9F8, intArrayOf(0x7B20)) + table.put(0x2F969, intArrayOf(0x7CE3)) + table.put(0xF9F7, intArrayOf(0x7ACB)) + table.put(0x2F968, intArrayOf(0x7CE8)) + table.put(0x1EB9, intArrayOf(0x0065, 0x0323)) + table.put(0xF9F6, intArrayOf(0x81E8)) + table.put(0x2F967, intArrayOf(0x42A0)) + table.put(0x1EB8, intArrayOf(0x0045, 0x0323)) + table.put(0xF9F5, intArrayOf(0x6DCB)) + table.put(0x2F966, intArrayOf(0x7CD2)) + table.put(0x1EB7, intArrayOf(0x0061, 0x0323, 0x0306)) + table.put(0xF9F4, intArrayOf(0x6797)) + table.put(0x2F965, intArrayOf(0x25C80)) + table.put(0x04D7, intArrayOf(0x0435, 0x0306)) + table.put(0x1EB6, intArrayOf(0x0041, 0x0323, 0x0306)) + table.put(0xF9F3, intArrayOf(0x9E9F)) + table.put(0x2F964, intArrayOf(0x4227)) + table.put(0x04D6, intArrayOf(0x0415, 0x0306)) + table.put(0x1EB5, intArrayOf(0x0061, 0x0306, 0x0303)) + table.put(0xF9F2, intArrayOf(0x9C57)) + table.put(0x2F963, intArrayOf(0x7BC9)) + table.put(0x1EB4, intArrayOf(0x0041, 0x0306, 0x0303)) + table.put(0xF9F1, intArrayOf(0x96A3)) + table.put(0x2F962, intArrayOf(0x7BC6)) + table.put(0x1EB3, intArrayOf(0x0061, 0x0306, 0x0309)) + table.put(0xF9F0, intArrayOf(0x85FA)) + table.put(0x2F961, intArrayOf(0x25BAB)) + table.put(0x04D3, intArrayOf(0x0430, 0x0308)) + table.put(0x1EB2, intArrayOf(0x0041, 0x0306, 0x0309)) + table.put(0x2F960, intArrayOf(0x4202)) + table.put(0x04D2, intArrayOf(0x0410, 0x0308)) + table.put(0x1EB1, intArrayOf(0x0061, 0x0306, 0x0300)) + table.put(0x04D1, intArrayOf(0x0430, 0x0306)) + table.put(0x1EB0, intArrayOf(0x0041, 0x0306, 0x0300)) + table.put(0x04D0, intArrayOf(0x0410, 0x0306)) + table.put(0xF9EF, intArrayOf(0x7498)) + table.put(0xF9EE, intArrayOf(0x71D0)) + table.put(0x2F95F, intArrayOf(0x7AEE)) + table.put(0xF9ED, intArrayOf(0x541D)) + table.put(0x2F95E, intArrayOf(0x25AA7)) + table.put(0x1EAF, intArrayOf(0x0061, 0x0306, 0x0301)) + table.put(0xF9EC, intArrayOf(0x6EBA)) + table.put(0x2F95D, intArrayOf(0x25AA7)) + table.put(0x1EAE, intArrayOf(0x0041, 0x0306, 0x0301)) + table.put(0xF9EB, intArrayOf(0x533F)) + table.put(0x2F95C, intArrayOf(0x2597C)) + table.put(0x1EAD, intArrayOf(0x0061, 0x0323, 0x0302)) + table.put(0xF9EA, intArrayOf(0x96E2)) + table.put(0x2F95B, intArrayOf(0x7A4F)) + table.put(0x1EAC, intArrayOf(0x0041, 0x0323, 0x0302)) + table.put(0x2F95A, intArrayOf(0x7A4A)) + table.put(0x1EAB, intArrayOf(0x0061, 0x0302, 0x0303)) + table.put(0x1EAA, intArrayOf(0x0041, 0x0302, 0x0303)) + table.put(0xF9E9, intArrayOf(0x91CC)) + table.put(0xF9E8, intArrayOf(0x88E1)) + table.put(0x2F959, intArrayOf(0x7A40)) + table.put(0xF9E7, intArrayOf(0x88CF)) + table.put(0x2F958, intArrayOf(0x412F)) + table.put(0x1EA9, intArrayOf(0x0061, 0x0302, 0x0309)) + table.put(0xF9E6, intArrayOf(0x7F79)) + table.put(0x2F957, intArrayOf(0x79EB)) + table.put(0x1EA8, intArrayOf(0x0041, 0x0302, 0x0309)) + table.put(0xF9E5, intArrayOf(0x75E2)) + table.put(0x2F956, intArrayOf(0x798F)) + table.put(0x1EA7, intArrayOf(0x0061, 0x0302, 0x0300)) + table.put(0xF9E4, intArrayOf(0x7406)) + table.put(0x2F955, intArrayOf(0x256C5)) + table.put(0x1EA6, intArrayOf(0x0041, 0x0302, 0x0300)) + table.put(0xF9E3, intArrayOf(0x6CE5)) + table.put(0x2F954, intArrayOf(0x2569A)) + table.put(0x1EA5, intArrayOf(0x0061, 0x0302, 0x0301)) + table.put(0xF9E2, intArrayOf(0x68A8)) + table.put(0x2F953, intArrayOf(0x7956)) + table.put(0x1EA4, intArrayOf(0x0041, 0x0302, 0x0301)) + table.put(0xF9E1, intArrayOf(0x674E)) + table.put(0x2F952, intArrayOf(0x25626)) + table.put(0x1EA3, intArrayOf(0x0061, 0x0309)) + table.put(0xF9E0, intArrayOf(0x6613)) + table.put(0x2F951, intArrayOf(0x40E3)) + table.put(0x1EA2, intArrayOf(0x0041, 0x0309)) + table.put(0x2F950, intArrayOf(0x78CC)) + table.put(0x04C2, intArrayOf(0x0436, 0x0306)) + table.put(0x1EA1, intArrayOf(0x0061, 0x0323)) + table.put(0x04C1, intArrayOf(0x0416, 0x0306)) + table.put(0x1EA0, intArrayOf(0x0041, 0x0323)) + table.put(0xF9DF, intArrayOf(0x5C65)) + table.put(0xF9DE, intArrayOf(0x540F)) + table.put(0x2F94F, intArrayOf(0x788C)) + table.put(0xF9DD, intArrayOf(0x5229)) + table.put(0x2F94E, intArrayOf(0x784E)) + table.put(0xF9DC, intArrayOf(0x9686)) + table.put(0x2F94D, intArrayOf(0x2541D)) + table.put(0xF9DB, intArrayOf(0x7387)) + table.put(0x2F94C, intArrayOf(0x4096)) + table.put(0xF9DA, intArrayOf(0x6817)) + table.put(0x2F94B, intArrayOf(0x4046)) + table.put(0x2F94A, intArrayOf(0x778B)) + table.put(0xF9D9, intArrayOf(0x6144)) + table.put(0xF9D8, intArrayOf(0x5F8B)) + table.put(0x2F949, intArrayOf(0x4039)) + table.put(0xF9D7, intArrayOf(0x8F2A)) + table.put(0x2F948, intArrayOf(0x774A)) + table.put(0xF9D6, intArrayOf(0x6DEA)) + table.put(0x2F947, intArrayOf(0x771F)) + table.put(0xF9D5, intArrayOf(0x5D19)) + table.put(0x2F946, intArrayOf(0x771F)) + table.put(0xF9D4, intArrayOf(0x502B)) + table.put(0x2F945, intArrayOf(0x771E)) + table.put(0xF9D3, intArrayOf(0x9678)) + table.put(0x2F944, intArrayOf(0x25133)) + table.put(0xF9D2, intArrayOf(0x622E)) + table.put(0x2F943, intArrayOf(0x25119)) + table.put(0xF9D1, intArrayOf(0x516D)) + table.put(0x2F942, intArrayOf(0x250F2)) + table.put(0xF9D0, intArrayOf(0x985E)) + table.put(0x2F941, intArrayOf(0x250F3)) + table.put(0x2F940, intArrayOf(0x76F4)) + table.put(0xF9CF, intArrayOf(0x7D10)) + table.put(0xF9CE, intArrayOf(0x786B)) + table.put(0x2F93F, intArrayOf(0x4008)) + table.put(0xF9CD, intArrayOf(0x7559)) + table.put(0x2F93E, intArrayOf(0x3FFC)) + table.put(0xF9CC, intArrayOf(0x7409)) + table.put(0x2F93D, intArrayOf(0x25044)) + table.put(0xF9CB, intArrayOf(0x6E9C)) + table.put(0x2F93C, intArrayOf(0x24FB8)) + table.put(0xF9CA, intArrayOf(0x6D41)) + table.put(0x2F93B, intArrayOf(0x24FA1)) + table.put(0x2F93A, intArrayOf(0x7610)) + table.put(0xF9C9, intArrayOf(0x67F3)) + table.put(0xF9C8, intArrayOf(0x677B)) + table.put(0x2F939, intArrayOf(0x2219F)) + table.put(0xF9C7, intArrayOf(0x5289)) + table.put(0x2F938, intArrayOf(0x7570)) + table.put(0xF9C6, intArrayOf(0x962E)) + table.put(0x2F937, intArrayOf(0x24C92)) + table.put(0xF9C5, intArrayOf(0x6688)) + table.put(0x2F936, intArrayOf(0x753E)) + table.put(0xF9C4, intArrayOf(0x9F8D)) + table.put(0x2F935, intArrayOf(0x24C36)) + table.put(0xF9C3, intArrayOf(0x907C)) + table.put(0x2F934, intArrayOf(0x7524)) + table.put(0xF9C2, intArrayOf(0x84FC)) + table.put(0x2F933, intArrayOf(0x3F1B)) + table.put(0xF9C1, intArrayOf(0x7642)) + table.put(0x2F932, intArrayOf(0x74CA)) + table.put(0xF9C0, intArrayOf(0x71CE)) + table.put(0x2F931, intArrayOf(0x7485)) + table.put(0x2F930, intArrayOf(0x7471)) + table.put(0xF9BF, intArrayOf(0x6A02)) + table.put(0xF9BE, intArrayOf(0x6599)) + table.put(0x2F92F, intArrayOf(0x745C)) + table.put(0xF9BD, intArrayOf(0x5C3F)) + table.put(0x2F92E, intArrayOf(0x7447)) + table.put(0xF9BC, intArrayOf(0x5BEE)) + table.put(0x2F92D, intArrayOf(0x3EB8)) + table.put(0xF9BB, intArrayOf(0x50DA)) + table.put(0x2F92C, intArrayOf(0x3EB8)) + table.put(0xF9BA, intArrayOf(0x4E86)) + table.put(0x2F92B, intArrayOf(0x73A5)) + table.put(0x2F92A, intArrayOf(0x3EAC)) + table.put(0xF9B9, intArrayOf(0x60E1)) + table.put(0xF9B8, intArrayOf(0x96B8)) + table.put(0x2F929, intArrayOf(0x738B)) + table.put(0xF9B7, intArrayOf(0x91B4)) + table.put(0x2F928, intArrayOf(0x737A)) + table.put(0xF9B6, intArrayOf(0x79AE)) + table.put(0x2F927, intArrayOf(0x24814)) + table.put(0xF9B5, intArrayOf(0x4F8B)) + table.put(0x2F926, intArrayOf(0x24735)) + table.put(0xF9B4, intArrayOf(0x9818)) + table.put(0x2F925, intArrayOf(0x7295)) + table.put(0xF9B3, intArrayOf(0x9748)) + table.put(0x2F924, intArrayOf(0x7280)) + table.put(0xF9B2, intArrayOf(0x96F6)) + table.put(0x2F923, intArrayOf(0x24608)) + table.put(0xF9B1, intArrayOf(0x9234)) + table.put(0x2F922, intArrayOf(0x7250)) + table.put(0xF9B0, intArrayOf(0x8046)) + table.put(0x2F921, intArrayOf(0x7235)) + table.put(0x2F920, intArrayOf(0x7228)) + table.put(0xF9AF, intArrayOf(0x7F9A)) + table.put(0xF9AE, intArrayOf(0x7469)) + table.put(0x2F91F, intArrayOf(0x243AB)) + table.put(0xF9AD, intArrayOf(0x73B2)) + table.put(0x2F91E, intArrayOf(0x719C)) + table.put(0xF9AC, intArrayOf(0x601C)) + table.put(0x2F91D, intArrayOf(0x24263)) + table.put(0xF9AB, intArrayOf(0x5DBA)) + table.put(0x2F91C, intArrayOf(0x7145)) + table.put(0xF9AA, intArrayOf(0x5BE7)) + table.put(0x2F91B, intArrayOf(0x20525)) + table.put(0x2F91A, intArrayOf(0x70AD)) + table.put(0xF9A9, intArrayOf(0x56F9)) + table.put(0xF9A8, intArrayOf(0x4EE4)) + table.put(0x2F919, intArrayOf(0x7077)) + table.put(0xF9A7, intArrayOf(0x7375)) + table.put(0x2F918, intArrayOf(0x707D)) + table.put(0xF9A6, intArrayOf(0x7C3E)) + table.put(0x2F917, intArrayOf(0x704A)) + table.put(0xF9A5, intArrayOf(0x6BAE)) + table.put(0x2F916, intArrayOf(0x3D96)) + table.put(0xF9A4, intArrayOf(0x637B)) + table.put(0x2F915, intArrayOf(0x701B)) + table.put(0xF9A3, intArrayOf(0x5FF5)) + table.put(0x2F914, intArrayOf(0x701E)) + table.put(0xF9A2, intArrayOf(0x5EC9)) + table.put(0x2F913, intArrayOf(0x7039)) + table.put(0xF9A1, intArrayOf(0x8AAA)) + table.put(0x2F912, intArrayOf(0x6FC6)) + table.put(0xF9A0, intArrayOf(0x88C2)) + table.put(0x2F911, intArrayOf(0x23F8E)) + table.put(0x2F910, intArrayOf(0x23F5E)) + table.put(0x2001, intArrayOf(0x2003)) + table.put(0x2000, intArrayOf(0x2002)) + table.put(0x2F90F, intArrayOf(0x6F6E)) + table.put(0x2F90E, intArrayOf(0x6DF9)) + table.put(0x2F90D, intArrayOf(0x23ED1)) + table.put(0x2F90C, intArrayOf(0x6EC7)) + table.put(0x2F90B, intArrayOf(0x6ECB)) + table.put(0x2F90A, intArrayOf(0x3D33)) + table.put(0x2F909, intArrayOf(0x6E6E)) + table.put(0x2F908, intArrayOf(0x6E2F)) + table.put(0x2F907, intArrayOf(0x6D34)) + table.put(0x2F906, intArrayOf(0x23D1E)) + table.put(0x2F905, intArrayOf(0x6D85)) + table.put(0x2F904, intArrayOf(0x6D78)) + table.put(0x2F903, intArrayOf(0x6D69)) + table.put(0x2F902, intArrayOf(0x6D41)) + table.put(0x2F901, intArrayOf(0x6D77)) + table.put(0x2F900, intArrayOf(0x6D3E)) + table.put(0x017E, intArrayOf(0x007A, 0x030C)) + table.put(0x017D, intArrayOf(0x005A, 0x030C)) + table.put(0x017C, intArrayOf(0x007A, 0x0307)) + table.put(0x017B, intArrayOf(0x005A, 0x0307)) + table.put(0x017A, intArrayOf(0x007A, 0x0301)) + table.put(0x0179, intArrayOf(0x005A, 0x0301)) + table.put(0x0178, intArrayOf(0x0059, 0x0308)) + table.put(0x0177, intArrayOf(0x0079, 0x0302)) + table.put(0x0176, intArrayOf(0x0059, 0x0302)) + table.put(0x0175, intArrayOf(0x0077, 0x0302)) + table.put(0x0174, intArrayOf(0x0057, 0x0302)) + table.put(0x0173, intArrayOf(0x0075, 0x0328)) + table.put(0x0172, intArrayOf(0x0055, 0x0328)) + table.put(0x0171, intArrayOf(0x0075, 0x030B)) + table.put(0x0170, intArrayOf(0x0055, 0x030B)) + table.put(0x016F, intArrayOf(0x0075, 0x030A)) + table.put(0x016E, intArrayOf(0x0055, 0x030A)) + table.put(0x016D, intArrayOf(0x0075, 0x0306)) + table.put(0x016C, intArrayOf(0x0055, 0x0306)) + table.put(0x016B, intArrayOf(0x0075, 0x0304)) + table.put(0x016A, intArrayOf(0x0055, 0x0304)) + table.put(0x0A5E, intArrayOf(0x0A2B, 0x0A3C)) + table.put(0x0169, intArrayOf(0x0075, 0x0303)) + table.put(0x0168, intArrayOf(0x0055, 0x0303)) + table.put(0x0A5B, intArrayOf(0x0A1C, 0x0A3C)) + table.put(0x0A5A, intArrayOf(0x0A17, 0x0A3C)) + table.put(0x0165, intArrayOf(0x0074, 0x030C)) + table.put(0x0164, intArrayOf(0x0054, 0x030C)) + table.put(0x0163, intArrayOf(0x0074, 0x0327)) + table.put(0x0162, intArrayOf(0x0054, 0x0327)) + table.put(0x0161, intArrayOf(0x0073, 0x030C)) + table.put(0x0160, intArrayOf(0x0053, 0x030C)) + table.put(0x0A59, intArrayOf(0x0A16, 0x0A3C)) + table.put(0x015F, intArrayOf(0x0073, 0x0327)) + table.put(0x015E, intArrayOf(0x0053, 0x0327)) + table.put(0x015D, intArrayOf(0x0073, 0x0302)) + table.put(0x015C, intArrayOf(0x0053, 0x0302)) + table.put(0x015B, intArrayOf(0x0073, 0x0301)) + table.put(0x015A, intArrayOf(0x0053, 0x0301)) + table.put(0x1E9B, intArrayOf(0x017F, 0x0307)) + table.put(0x0159, intArrayOf(0x0072, 0x030C)) + table.put(0x0158, intArrayOf(0x0052, 0x030C)) + table.put(0x1E99, intArrayOf(0x0079, 0x030A)) + table.put(0x232A, intArrayOf(0x3009)) + table.put(0x0157, intArrayOf(0x0072, 0x0327)) + table.put(0x1E98, intArrayOf(0x0077, 0x030A)) + table.put(0x0156, intArrayOf(0x0052, 0x0327)) + table.put(0x1E97, intArrayOf(0x0074, 0x0308)) + table.put(0x0155, intArrayOf(0x0072, 0x0301)) + table.put(0x1E96, intArrayOf(0x0068, 0x0331)) + table.put(0x0154, intArrayOf(0x0052, 0x0301)) + table.put(0x1E95, intArrayOf(0x007A, 0x0331)) + table.put(0x1E94, intArrayOf(0x005A, 0x0331)) + table.put(0x1E93, intArrayOf(0x007A, 0x0323)) + table.put(0x0151, intArrayOf(0x006F, 0x030B)) + table.put(0x1E92, intArrayOf(0x005A, 0x0323)) + table.put(0x0150, intArrayOf(0x004F, 0x030B)) + table.put(0x1E91, intArrayOf(0x007A, 0x0302)) + table.put(0x2329, intArrayOf(0x3008)) + table.put(0x1E90, intArrayOf(0x005A, 0x0302)) + table.put(0x014F, intArrayOf(0x006F, 0x0306)) + table.put(0x014E, intArrayOf(0x004F, 0x0306)) + table.put(0x1E8F, intArrayOf(0x0079, 0x0307)) + table.put(0x014D, intArrayOf(0x006F, 0x0304)) + table.put(0x1E8E, intArrayOf(0x0059, 0x0307)) + table.put(0x014C, intArrayOf(0x004F, 0x0304)) + table.put(0x1E8D, intArrayOf(0x0078, 0x0308)) + table.put(0x1E8C, intArrayOf(0x0058, 0x0308)) + table.put(0x1E8B, intArrayOf(0x0078, 0x0307)) + table.put(0x1E8A, intArrayOf(0x0058, 0x0307)) + table.put(0x0148, intArrayOf(0x006E, 0x030C)) + table.put(0x1E89, intArrayOf(0x0077, 0x0323)) + table.put(0x0147, intArrayOf(0x004E, 0x030C)) + table.put(0x1E88, intArrayOf(0x0057, 0x0323)) + table.put(0x0146, intArrayOf(0x006E, 0x0327)) + table.put(0x1E87, intArrayOf(0x0077, 0x0307)) + table.put(0x0145, intArrayOf(0x004E, 0x0327)) + table.put(0x1E86, intArrayOf(0x0057, 0x0307)) + table.put(0x0144, intArrayOf(0x006E, 0x0301)) + table.put(0x1E85, intArrayOf(0x0077, 0x0308)) + table.put(0x0143, intArrayOf(0x004E, 0x0301)) + table.put(0x1E84, intArrayOf(0x0057, 0x0308)) + table.put(0x1E83, intArrayOf(0x0077, 0x0301)) + table.put(0x1E82, intArrayOf(0x0057, 0x0301)) + table.put(0x1E81, intArrayOf(0x0077, 0x0300)) + table.put(0x1E80, intArrayOf(0x0057, 0x0300)) + table.put(0x0A36, intArrayOf(0x0A38, 0x0A3C)) + table.put(0x0A33, intArrayOf(0x0A32, 0x0A3C)) + table.put(0x013E, intArrayOf(0x006C, 0x030C)) + table.put(0x1E7F, intArrayOf(0x0076, 0x0323)) + table.put(0x013D, intArrayOf(0x004C, 0x030C)) + table.put(0x1E7E, intArrayOf(0x0056, 0x0323)) + table.put(0x013C, intArrayOf(0x006C, 0x0327)) + table.put(0x1E7D, intArrayOf(0x0076, 0x0303)) + table.put(0x013B, intArrayOf(0x004C, 0x0327)) + table.put(0x1E7C, intArrayOf(0x0056, 0x0303)) + table.put(0x013A, intArrayOf(0x006C, 0x0301)) + table.put(0x1E7B, intArrayOf(0x0075, 0x0304, 0x0308)) + table.put(0x1E7A, intArrayOf(0x0055, 0x0304, 0x0308)) + table.put(0x0139, intArrayOf(0x004C, 0x0301)) + table.put(0x1E79, intArrayOf(0x0075, 0x0303, 0x0301)) + table.put(0x0137, intArrayOf(0x006B, 0x0327)) + table.put(0x1E78, intArrayOf(0x0055, 0x0303, 0x0301)) + table.put(0x0136, intArrayOf(0x004B, 0x0327)) + table.put(0x1E77, intArrayOf(0x0075, 0x032D)) + table.put(0x0135, intArrayOf(0x006A, 0x0302)) + table.put(0x1E76, intArrayOf(0x0055, 0x032D)) + table.put(0x0134, intArrayOf(0x004A, 0x0302)) + table.put(0x1E75, intArrayOf(0x0075, 0x0330)) + table.put(0x1E74, intArrayOf(0x0055, 0x0330)) + table.put(0x1E73, intArrayOf(0x0075, 0x0324)) + table.put(0x1E72, intArrayOf(0x0055, 0x0324)) + table.put(0x0130, intArrayOf(0x0049, 0x0307)) + table.put(0x1E71, intArrayOf(0x0074, 0x032D)) + table.put(0x1E70, intArrayOf(0x0054, 0x032D)) + table.put(0x012F, intArrayOf(0x0069, 0x0328)) + table.put(0x012E, intArrayOf(0x0049, 0x0328)) + table.put(0x1E6F, intArrayOf(0x0074, 0x0331)) + table.put(0x012D, intArrayOf(0x0069, 0x0306)) + table.put(0x1E6E, intArrayOf(0x0054, 0x0331)) + table.put(0x012C, intArrayOf(0x0049, 0x0306)) + table.put(0x1E6D, intArrayOf(0x0074, 0x0323)) + table.put(0x012B, intArrayOf(0x0069, 0x0304)) + table.put(0x1E6C, intArrayOf(0x0054, 0x0323)) + table.put(0x012A, intArrayOf(0x0049, 0x0304)) + table.put(0x1E6B, intArrayOf(0x0074, 0x0307)) + table.put(0x1E6A, intArrayOf(0x0054, 0x0307)) + table.put(0x0129, intArrayOf(0x0069, 0x0303)) + table.put(0x0128, intArrayOf(0x0049, 0x0303)) + table.put(0x1E69, intArrayOf(0x0073, 0x0323, 0x0307)) + table.put(0x1E68, intArrayOf(0x0053, 0x0323, 0x0307)) + table.put(0x1E67, intArrayOf(0x0073, 0x030C, 0x0307)) + table.put(0x0125, intArrayOf(0x0068, 0x0302)) + table.put(0x1E66, intArrayOf(0x0053, 0x030C, 0x0307)) + table.put(0x0124, intArrayOf(0x0048, 0x0302)) + table.put(0x1E65, intArrayOf(0x0073, 0x0301, 0x0307)) + table.put(0x0123, intArrayOf(0x0067, 0x0327)) + table.put(0x1E64, intArrayOf(0x0053, 0x0301, 0x0307)) + table.put(0x0122, intArrayOf(0x0047, 0x0327)) + table.put(0x1E63, intArrayOf(0x0073, 0x0323)) + table.put(0x0121, intArrayOf(0x0067, 0x0307)) + table.put(0x1E62, intArrayOf(0x0053, 0x0323)) + table.put(0x0120, intArrayOf(0x0047, 0x0307)) + table.put(0x1E61, intArrayOf(0x0073, 0x0307)) + table.put(0x1E60, intArrayOf(0x0053, 0x0307)) + table.put(0x30FE, intArrayOf(0x30FD, 0x3099)) + table.put(0xF99F, intArrayOf(0x70C8)) + table.put(0xF99E, intArrayOf(0x54BD)) + table.put(0x011F, intArrayOf(0x0067, 0x0306)) + table.put(0xF99D, intArrayOf(0x52A3)) + table.put(0x011E, intArrayOf(0x0047, 0x0306)) + table.put(0x1E5F, intArrayOf(0x0072, 0x0331)) + table.put(0x30FA, intArrayOf(0x30F2, 0x3099)) + table.put(0xF99C, intArrayOf(0x5217)) + table.put(0x011D, intArrayOf(0x0067, 0x0302)) + table.put(0x1E5E, intArrayOf(0x0052, 0x0331)) + table.put(0xF99B, intArrayOf(0x934A)) + table.put(0x011C, intArrayOf(0x0047, 0x0302)) + table.put(0x1E5D, intArrayOf(0x0072, 0x0323, 0x0304)) + table.put(0xF99A, intArrayOf(0x9023)) + table.put(0x011B, intArrayOf(0x0065, 0x030C)) + table.put(0x1E5C, intArrayOf(0x0052, 0x0323, 0x0304)) + table.put(0x011A, intArrayOf(0x0045, 0x030C)) + table.put(0x1E5B, intArrayOf(0x0072, 0x0323)) + table.put(0x1E5A, intArrayOf(0x0052, 0x0323)) + table.put(0x30F9, intArrayOf(0x30F1, 0x3099)) + table.put(0x30F8, intArrayOf(0x30F0, 0x3099)) + table.put(0x30F7, intArrayOf(0x30EF, 0x3099)) + table.put(0xF999, intArrayOf(0x84EE)) + table.put(0xF998, intArrayOf(0x8F26)) + table.put(0x0119, intArrayOf(0x0065, 0x0328)) + table.put(0xF997, intArrayOf(0x806F)) + table.put(0x0118, intArrayOf(0x0045, 0x0328)) + table.put(0x1E59, intArrayOf(0x0072, 0x0307)) + table.put(0x30F4, intArrayOf(0x30A6, 0x3099)) + table.put(0xF996, intArrayOf(0x7DF4)) + table.put(0x0117, intArrayOf(0x0065, 0x0307)) + table.put(0x1E58, intArrayOf(0x0052, 0x0307)) + table.put(0xF995, intArrayOf(0x79CA)) + table.put(0x0116, intArrayOf(0x0045, 0x0307)) + table.put(0x1E57, intArrayOf(0x0070, 0x0307)) + table.put(0xF994, intArrayOf(0x7489)) + table.put(0xF993, intArrayOf(0x7149)) + table.put(0x0115, intArrayOf(0x0065, 0x0306)) + table.put(0x0477, intArrayOf(0x0475, 0x030F)) + table.put(0x1E56, intArrayOf(0x0050, 0x0307)) + table.put(0xF992, intArrayOf(0x6F23)) + table.put(0x1E55, intArrayOf(0x0070, 0x0301)) + table.put(0x0476, intArrayOf(0x0474, 0x030F)) + table.put(0x0114, intArrayOf(0x0045, 0x0306)) + table.put(0xF991, intArrayOf(0x649A)) + table.put(0x1E54, intArrayOf(0x0050, 0x0301)) + table.put(0x0113, intArrayOf(0x0065, 0x0304)) + table.put(0xF990, intArrayOf(0x6200)) + table.put(0x1E53, intArrayOf(0x006F, 0x0304, 0x0301)) + table.put(0x0112, intArrayOf(0x0045, 0x0304)) + table.put(0x1E52, intArrayOf(0x004F, 0x0304, 0x0301)) + table.put(0x1E51, intArrayOf(0x006F, 0x0304, 0x0300)) + table.put(0x1E50, intArrayOf(0x004F, 0x0304, 0x0300)) + table.put(0xF98F, intArrayOf(0x6190)) + table.put(0xF98E, intArrayOf(0x5E74)) + table.put(0xF98D, intArrayOf(0x8F62)) + table.put(0x010F, intArrayOf(0x0064, 0x030C)) + table.put(0xF98C, intArrayOf(0x6B77)) + table.put(0x1E4F, intArrayOf(0x006F, 0x0303, 0x0308)) + table.put(0x010E, intArrayOf(0x0044, 0x030C)) + table.put(0xF98B, intArrayOf(0x66C6)) + table.put(0x1E4E, intArrayOf(0x004F, 0x0303, 0x0308)) + table.put(0x010D, intArrayOf(0x0063, 0x030C)) + table.put(0xF98A, intArrayOf(0x529B)) + table.put(0x1E4D, intArrayOf(0x006F, 0x0303, 0x0301)) + table.put(0x010C, intArrayOf(0x0043, 0x030C)) + table.put(0x1E4C, intArrayOf(0x004F, 0x0303, 0x0301)) + table.put(0x010B, intArrayOf(0x0063, 0x0307)) + table.put(0x1E4B, intArrayOf(0x006E, 0x032D)) + table.put(0x010A, intArrayOf(0x0043, 0x0307)) + table.put(0x1E4A, intArrayOf(0x004E, 0x032D)) + table.put(0xF989, intArrayOf(0x9ECE)) + table.put(0xF988, intArrayOf(0x9E97)) + table.put(0xF987, intArrayOf(0x9A6A)) + table.put(0x0109, intArrayOf(0x0063, 0x0302)) + table.put(0xF986, intArrayOf(0x95AD)) + table.put(0x1E49, intArrayOf(0x006E, 0x0331)) + table.put(0x0108, intArrayOf(0x0043, 0x0302)) + table.put(0xF985, intArrayOf(0x792A)) + table.put(0x1E48, intArrayOf(0x004E, 0x0331)) + table.put(0x0107, intArrayOf(0x0063, 0x0301)) + table.put(0xF984, intArrayOf(0x6FFE)) + table.put(0x1E47, intArrayOf(0x006E, 0x0323)) + table.put(0x0106, intArrayOf(0x0043, 0x0301)) + table.put(0xF983, intArrayOf(0x65C5)) + table.put(0x1E46, intArrayOf(0x004E, 0x0323)) + table.put(0x0105, intArrayOf(0x0061, 0x0328)) + table.put(0xF982, intArrayOf(0x5EEC)) + table.put(0x1E45, intArrayOf(0x006E, 0x0307)) + table.put(0x0104, intArrayOf(0x0041, 0x0328)) + table.put(0xF981, intArrayOf(0x5973)) + table.put(0x1E44, intArrayOf(0x004E, 0x0307)) + table.put(0x0103, intArrayOf(0x0061, 0x0306)) + table.put(0xF980, intArrayOf(0x5442)) + table.put(0x1E43, intArrayOf(0x006D, 0x0323)) + table.put(0x0102, intArrayOf(0x0041, 0x0306)) + table.put(0x1E42, intArrayOf(0x004D, 0x0323)) + table.put(0x0101, intArrayOf(0x0061, 0x0304)) + table.put(0x1E41, intArrayOf(0x006D, 0x0307)) + table.put(0x0100, intArrayOf(0x0041, 0x0304)) + table.put(0x1E40, intArrayOf(0x004D, 0x0307)) + table.put(0xF97F, intArrayOf(0x52F5)) + table.put(0x30DD, intArrayOf(0x30DB, 0x309A)) + table.put(0xF97E, intArrayOf(0x91CF)) + table.put(0x30DC, intArrayOf(0x30DB, 0x3099)) + table.put(0xF97D, intArrayOf(0x8AD2)) + table.put(0x2F8FF, intArrayOf(0x6D16)) + table.put(0xF97C, intArrayOf(0x826F)) + table.put(0x30DA, intArrayOf(0x30D8, 0x309A)) + table.put(0x1E3F, intArrayOf(0x006D, 0x0301)) + table.put(0x2F8FE, intArrayOf(0x6C67)) + table.put(0xF97B, intArrayOf(0x7CE7)) + table.put(0x1E3E, intArrayOf(0x004D, 0x0301)) + table.put(0x2F8FD, intArrayOf(0x6CCD)) + table.put(0xF97A, intArrayOf(0x6881)) + table.put(0x1E3D, intArrayOf(0x006C, 0x032D)) + table.put(0x045E, intArrayOf(0x0443, 0x0306)) + table.put(0x2F8FC, intArrayOf(0x6CBF)) + table.put(0x1E3C, intArrayOf(0x004C, 0x032D)) + table.put(0x045D, intArrayOf(0x0438, 0x0300)) + table.put(0x2F8FB, intArrayOf(0x23CBC)) + table.put(0x1E3B, intArrayOf(0x006C, 0x0331)) + table.put(0x045C, intArrayOf(0x043A, 0x0301)) + table.put(0x2F8FA, intArrayOf(0x6C4E)) + table.put(0x1E3A, intArrayOf(0x004C, 0x0331)) + table.put(0x30D9, intArrayOf(0x30D8, 0x3099)) + table.put(0xF979, intArrayOf(0x51C9)) + table.put(0x30D7, intArrayOf(0x30D5, 0x309A)) + table.put(0xF978, intArrayOf(0x5169)) + table.put(0x30D6, intArrayOf(0x30D5, 0x3099)) + table.put(0xF977, intArrayOf(0x4EAE)) + table.put(0x2F8F9, intArrayOf(0x23AFA)) + table.put(0xF976, intArrayOf(0x7565)) + table.put(0x30D4, intArrayOf(0x30D2, 0x309A)) + table.put(0x1E39, intArrayOf(0x006C, 0x0323, 0x0304)) + table.put(0x2F8F8, intArrayOf(0x21D0B)) + table.put(0xF975, intArrayOf(0x63A0)) + table.put(0x30D3, intArrayOf(0x30D2, 0x3099)) + table.put(0x1E38, intArrayOf(0x004C, 0x0323, 0x0304)) + table.put(0x0D4C, intArrayOf(0x0D46, 0x0D57)) + table.put(0x2F8F7, intArrayOf(0x23A8D)) + table.put(0xF974, intArrayOf(0x82E5)) + table.put(0x1E37, intArrayOf(0x006C, 0x0323)) + table.put(0x0D4B, intArrayOf(0x0D47, 0x0D3E)) + table.put(0x2F8F6, intArrayOf(0x6BBB)) + table.put(0xF973, intArrayOf(0x62FE)) + table.put(0x30D1, intArrayOf(0x30CF, 0x309A)) + table.put(0x1E36, intArrayOf(0x004C, 0x0323)) + table.put(0x0D4A, intArrayOf(0x0D46, 0x0D3E)) + table.put(0x0457, intArrayOf(0x0456, 0x0308)) + table.put(0x2F8F5, intArrayOf(0x6BBA)) + table.put(0xF972, intArrayOf(0x6C88)) + table.put(0x30D0, intArrayOf(0x30CF, 0x3099)) + table.put(0x1E35, intArrayOf(0x006B, 0x0331)) + table.put(0x2F8F4, intArrayOf(0x6B9F)) + table.put(0xF971, intArrayOf(0x8FB0)) + table.put(0x1E34, intArrayOf(0x004B, 0x0331)) + table.put(0x2F8F3, intArrayOf(0x6B72)) + table.put(0xF970, intArrayOf(0x6BBA)) + table.put(0x1E33, intArrayOf(0x006B, 0x0323)) + table.put(0x2F8F2, intArrayOf(0x3C4E)) + table.put(0x1E32, intArrayOf(0x004B, 0x0323)) + table.put(0x0453, intArrayOf(0x0433, 0x0301)) + table.put(0x2F8F1, intArrayOf(0x6B54)) + table.put(0x1E31, intArrayOf(0x006B, 0x0301)) + table.put(0x2F8F0, intArrayOf(0x238A7)) + table.put(0x1E30, intArrayOf(0x004B, 0x0301)) + table.put(0x0451, intArrayOf(0x0435, 0x0308)) + table.put(0x0450, intArrayOf(0x0435, 0x0300)) + table.put(0xF96F, intArrayOf(0x8AAA)) + table.put(0xF96E, intArrayOf(0x8449)) + table.put(0xF96D, intArrayOf(0x7701)) + table.put(0x2F8EF, intArrayOf(0x6B21)) + table.put(0xF96C, intArrayOf(0x585E)) + table.put(0x1E2F, intArrayOf(0x0069, 0x0308, 0x0301)) + table.put(0x2F8EE, intArrayOf(0x3C18)) + table.put(0xF96B, intArrayOf(0x53C3)) + table.put(0x1E2E, intArrayOf(0x0049, 0x0308, 0x0301)) + table.put(0x2F8ED, intArrayOf(0x6ADB)) + table.put(0xF96A, intArrayOf(0x7D22)) + table.put(0x1E2D, intArrayOf(0x0069, 0x0330)) + table.put(0x2F8EC, intArrayOf(0x236A3)) + table.put(0x1E2C, intArrayOf(0x0049, 0x0330)) + table.put(0x2F8EB, intArrayOf(0x6AA8)) + table.put(0x1E2B, intArrayOf(0x0068, 0x032E)) + table.put(0x2F8EA, intArrayOf(0x69EA)) + table.put(0x1E2A, intArrayOf(0x0048, 0x032E)) + table.put(0x30C9, intArrayOf(0x30C8, 0x3099)) + table.put(0xF969, intArrayOf(0x6578)) + table.put(0x30C7, intArrayOf(0x30C6, 0x3099)) + table.put(0xF968, intArrayOf(0x6CCC)) + table.put(0xF967, intArrayOf(0x4E0D)) + table.put(0x30C5, intArrayOf(0x30C4, 0x3099)) + table.put(0x2F8E9, intArrayOf(0x69A3)) + table.put(0xF966, intArrayOf(0x5FA9)) + table.put(0x1E29, intArrayOf(0x0068, 0x0327)) + table.put(0x2F8E8, intArrayOf(0x6942)) + table.put(0xF965, intArrayOf(0x4FBF)) + table.put(0x1E28, intArrayOf(0x0048, 0x0327)) + table.put(0x2F8E7, intArrayOf(0x3B9D)) + table.put(0xF964, intArrayOf(0x78FB)) + table.put(0x30C2, intArrayOf(0x30C1, 0x3099)) + table.put(0x1E27, intArrayOf(0x0068, 0x0308)) + table.put(0x2F8E6, intArrayOf(0x6914)) + table.put(0xF963, intArrayOf(0x5317)) + table.put(0x1E26, intArrayOf(0x0048, 0x0308)) + table.put(0x2F8E5, intArrayOf(0x681F)) + table.put(0xF962, intArrayOf(0x7570)) + table.put(0x30C0, intArrayOf(0x30BF, 0x3099)) + table.put(0x1E25, intArrayOf(0x0068, 0x0323)) + table.put(0x2F8E4, intArrayOf(0x688E)) + table.put(0xF961, intArrayOf(0x7387)) + table.put(0x1E24, intArrayOf(0x0048, 0x0323)) + table.put(0x2F8E3, intArrayOf(0x2346D)) + table.put(0xF960, intArrayOf(0x6012)) + table.put(0x1E23, intArrayOf(0x0068, 0x0307)) + table.put(0x2F8E2, intArrayOf(0x6885)) + table.put(0x1E22, intArrayOf(0x0048, 0x0307)) + table.put(0x2F8E1, intArrayOf(0x6852)) + table.put(0x1E21, intArrayOf(0x0067, 0x0304)) + table.put(0x2F8E0, intArrayOf(0x6785)) + table.put(0x1E20, intArrayOf(0x0047, 0x0304)) + table.put(0x30BE, intArrayOf(0x30BD, 0x3099)) + table.put(0xF95F, intArrayOf(0x5BE7)) + table.put(0xF95E, intArrayOf(0x4E39)) + table.put(0x30BC, intArrayOf(0x30BB, 0x3099)) + table.put(0xF95D, intArrayOf(0x8AFE)) + table.put(0x2F8DF, intArrayOf(0x67FA)) + table.put(0xF95C, intArrayOf(0x6A02)) + table.put(0x30BA, intArrayOf(0x30B9, 0x3099)) + table.put(0x1E1F, intArrayOf(0x0066, 0x0307)) + table.put(0x2F8DE, intArrayOf(0x3B49)) + table.put(0xF95B, intArrayOf(0x62CF)) + table.put(0x1E1E, intArrayOf(0x0046, 0x0307)) + table.put(0x2F8DD, intArrayOf(0x233C3)) + table.put(0xF95A, intArrayOf(0x8B80)) + table.put(0x1E1D, intArrayOf(0x0065, 0x0327, 0x0306)) + table.put(0x2F8DC, intArrayOf(0x6753)) + table.put(0x1E1C, intArrayOf(0x0045, 0x0327, 0x0306)) + table.put(0x2F8DB, intArrayOf(0x675E)) + table.put(0x1E1B, intArrayOf(0x0065, 0x0330)) + table.put(0x2F8DA, intArrayOf(0x6721)) + table.put(0x1E1A, intArrayOf(0x0045, 0x0330)) + table.put(0x30B8, intArrayOf(0x30B7, 0x3099)) + table.put(0xF959, intArrayOf(0x9675)) + table.put(0xF958, intArrayOf(0x83F1)) + table.put(0x30B6, intArrayOf(0x30B5, 0x3099)) + table.put(0xF957, intArrayOf(0x7DBE)) + table.put(0x2F8D9, intArrayOf(0x671B)) + table.put(0xF956, intArrayOf(0x7A1C)) + table.put(0x30B4, intArrayOf(0x30B3, 0x3099)) + table.put(0x1E19, intArrayOf(0x0065, 0x032D)) + table.put(0x2F8D8, intArrayOf(0x6717)) + table.put(0xF955, intArrayOf(0x51CC)) + table.put(0x1E18, intArrayOf(0x0045, 0x032D)) + table.put(0x0439, intArrayOf(0x0438, 0x0306)) + table.put(0x2F8D7, intArrayOf(0x43D9)) + table.put(0xF954, intArrayOf(0x51DC)) + table.put(0x30B2, intArrayOf(0x30B1, 0x3099)) + table.put(0x1E17, intArrayOf(0x0065, 0x0304, 0x0301)) + table.put(0x2F8D6, intArrayOf(0x80AD)) + table.put(0xF953, intArrayOf(0x808B)) + table.put(0x1E16, intArrayOf(0x0045, 0x0304, 0x0301)) + table.put(0x2F8D5, intArrayOf(0x669C)) + table.put(0xF952, intArrayOf(0x52D2)) + table.put(0x30B0, intArrayOf(0x30AF, 0x3099)) + table.put(0x1E15, intArrayOf(0x0065, 0x0304, 0x0300)) + table.put(0x2F8D4, intArrayOf(0x6700)) + table.put(0xF951, intArrayOf(0x964B)) + table.put(0x1E14, intArrayOf(0x0045, 0x0304, 0x0300)) + table.put(0x2F8D3, intArrayOf(0x5195)) + table.put(0xF950, intArrayOf(0x7E37)) + table.put(0x1E13, intArrayOf(0x0064, 0x032D)) + table.put(0x2F8D2, intArrayOf(0x5192)) + table.put(0x1E12, intArrayOf(0x0044, 0x032D)) + table.put(0x2F8D1, intArrayOf(0x3AE4)) + table.put(0x1E11, intArrayOf(0x0064, 0x0327)) + table.put(0x2F8D0, intArrayOf(0x3B08)) + table.put(0x1E10, intArrayOf(0x0044, 0x0327)) + table.put(0x30AE, intArrayOf(0x30AD, 0x3099)) + table.put(0xF94F, intArrayOf(0x7D2F)) + table.put(0xF94E, intArrayOf(0x6F0F)) + table.put(0x30AC, intArrayOf(0x30AB, 0x3099)) + table.put(0xF94D, intArrayOf(0x6DDA)) + table.put(0x2F8CF, intArrayOf(0x6691)) + table.put(0xF94C, intArrayOf(0x6A13)) + table.put(0x1E0F, intArrayOf(0x0064, 0x0331)) + table.put(0x2F8CE, intArrayOf(0x3B19)) + table.put(0xF94B, intArrayOf(0x5C62)) + table.put(0x1E0E, intArrayOf(0x0044, 0x0331)) + table.put(0x2F8CD, intArrayOf(0x6649)) + table.put(0xF94A, intArrayOf(0x58D8)) + table.put(0x1E0D, intArrayOf(0x0064, 0x0323)) + table.put(0x2F8CC, intArrayOf(0x66F8)) + table.put(0x1E0C, intArrayOf(0x0044, 0x0323)) + table.put(0x2F8CB, intArrayOf(0x65E3)) + table.put(0x1E0B, intArrayOf(0x0064, 0x0307)) + table.put(0x2F8CA, intArrayOf(0x2300A)) + table.put(0x1E0A, intArrayOf(0x0044, 0x0307)) + table.put(0xF949, intArrayOf(0x96F7)) + table.put(0xF948, intArrayOf(0x8CC2)) + table.put(0xF947, intArrayOf(0x78CA)) + table.put(0x2F8C9, intArrayOf(0x656C)) + table.put(0xF946, intArrayOf(0x7262)) + table.put(0x1E09, intArrayOf(0x0063, 0x0327, 0x0301)) + table.put(0x2F8C8, intArrayOf(0x654F)) + table.put(0xF945, intArrayOf(0x807E)) + table.put(0x1E08, intArrayOf(0x0043, 0x0327, 0x0301)) + table.put(0x2F8C7, intArrayOf(0x3A6C)) + table.put(0xF944, intArrayOf(0x7C60)) + table.put(0x1E07, intArrayOf(0x0062, 0x0331)) + table.put(0x2F8C6, intArrayOf(0x6477)) + table.put(0xF943, intArrayOf(0x5F04)) + table.put(0x1E06, intArrayOf(0x0042, 0x0331)) + table.put(0x2F8C5, intArrayOf(0x649D)) + table.put(0xF942, intArrayOf(0x58DF)) + table.put(0x1E05, intArrayOf(0x0062, 0x0323)) + table.put(0x2F8C4, intArrayOf(0x647E)) + table.put(0xF941, intArrayOf(0x8AD6)) + table.put(0x1E04, intArrayOf(0x0042, 0x0323)) + table.put(0x2F8C3, intArrayOf(0x6469)) + table.put(0xF940, intArrayOf(0x9E7F)) + table.put(0x1E03, intArrayOf(0x0062, 0x0307)) + table.put(0x2F8C2, intArrayOf(0x3A2E)) + table.put(0x1E02, intArrayOf(0x0042, 0x0307)) + table.put(0x2F8C1, intArrayOf(0x63A9)) + table.put(0x1E01, intArrayOf(0x0061, 0x0325)) + table.put(0x2F8C0, intArrayOf(0x63C5)) + table.put(0x1E00, intArrayOf(0x0041, 0x0325)) + table.put(0xF93F, intArrayOf(0x9304)) + table.put(0xF93E, intArrayOf(0x83C9)) + table.put(0xF93D, intArrayOf(0x7DA0)) + table.put(0x2F8BF, intArrayOf(0x6422)) + table.put(0xF93C, intArrayOf(0x797F)) + table.put(0x2F8BE, intArrayOf(0x22BF1)) + table.put(0xF93B, intArrayOf(0x788C)) + table.put(0x2F8BD, intArrayOf(0x63E4)) + table.put(0xF93A, intArrayOf(0x9DFA)) + table.put(0x2F8BC, intArrayOf(0x6383)) + table.put(0x2F8BB, intArrayOf(0x6368)) + table.put(0x2F8BA, intArrayOf(0x62FC)) + table.put(0xF939, intArrayOf(0x9B6F)) + table.put(0xF938, intArrayOf(0x9732)) + table.put(0xF937, intArrayOf(0x8DEF)) + table.put(0x2F8B9, intArrayOf(0x633D)) + table.put(0xF936, intArrayOf(0x865C)) + table.put(0x2F8B8, intArrayOf(0x22B0C)) + table.put(0xF935, intArrayOf(0x8606)) + table.put(0x0419, intArrayOf(0x0418, 0x0306)) + table.put(0x2F8B7, intArrayOf(0x6350)) + table.put(0xF934, intArrayOf(0x8001)) + table.put(0x2F8B6, intArrayOf(0x62D4)) + table.put(0xF933, intArrayOf(0x76E7)) + table.put(0x2F8B5, intArrayOf(0x62B1)) + table.put(0xF932, intArrayOf(0x7210)) + table.put(0x2F8B4, intArrayOf(0x625D)) + table.put(0xF931, intArrayOf(0x6AD3)) + table.put(0x2F8B3, intArrayOf(0x621B)) + table.put(0xF930, intArrayOf(0x64C4)) + table.put(0x2F8B2, intArrayOf(0x6210)) + table.put(0x2F8B1, intArrayOf(0x61F6)) + table.put(0x2F8B0, intArrayOf(0x61F2)) + table.put(0xF92F, intArrayOf(0x52DE)) + table.put(0xF92E, intArrayOf(0x51B7)) + table.put(0xF92D, intArrayOf(0x4F86)) + table.put(0x2F8AF, intArrayOf(0x61DE)) + table.put(0xF92C, intArrayOf(0x90CE)) + table.put(0x2F8AE, intArrayOf(0x61AF)) + table.put(0xF92B, intArrayOf(0x72FC)) + table.put(0x2F8AD, intArrayOf(0x61A4)) + table.put(0xF92A, intArrayOf(0x6D6A)) + table.put(0x040E, intArrayOf(0x0423, 0x0306)) + table.put(0x2F8AC, intArrayOf(0x61B2)) + table.put(0x040D, intArrayOf(0x0418, 0x0300)) + table.put(0x2F8AB, intArrayOf(0x618E)) + table.put(0x040C, intArrayOf(0x041A, 0x0301)) + table.put(0x2F8AA, intArrayOf(0x617A)) + table.put(0xF929, intArrayOf(0x6717)) + table.put(0xF928, intArrayOf(0x5ECA)) + table.put(0xF927, intArrayOf(0x881F)) + table.put(0x2F8A9, intArrayOf(0x614C)) + table.put(0xF926, intArrayOf(0x81D8)) + table.put(0x2F8A8, intArrayOf(0x614E)) + table.put(0xF925, intArrayOf(0x62C9)) + table.put(0x2F8A7, intArrayOf(0x614C)) + table.put(0xF924, intArrayOf(0x8964)) + table.put(0x2F8A6, intArrayOf(0x6148)) + table.put(0xF923, intArrayOf(0x85CD)) + table.put(0x0407, intArrayOf(0x0406, 0x0308)) + table.put(0x2F8A5, intArrayOf(0x60C7)) + table.put(0xF922, intArrayOf(0x6FEB)) + table.put(0x2F8A4, intArrayOf(0x226D4)) + table.put(0xF921, intArrayOf(0x5D50)) + table.put(0x2F8A3, intArrayOf(0x6094)) + table.put(0xF920, intArrayOf(0x9E1E)) + table.put(0x2F8A2, intArrayOf(0x391C)) + table.put(0x0403, intArrayOf(0x0413, 0x0301)) + table.put(0x2F8A1, intArrayOf(0x393A)) + table.put(0x2F8A0, intArrayOf(0x6081)) + table.put(0x0401, intArrayOf(0x0415, 0x0308)) + table.put(0x0400, intArrayOf(0x0415, 0x0300)) + table.put(0xF91F, intArrayOf(0x862D)) + table.put(0xF91E, intArrayOf(0x721B)) + table.put(0xF91D, intArrayOf(0x6B04)) + table.put(0xF91C, intArrayOf(0x5375)) + table.put(0xF91B, intArrayOf(0x4E82)) + table.put(0xF91A, intArrayOf(0x99F1)) + table.put(0xF919, intArrayOf(0x916A)) + table.put(0x22ED, intArrayOf(0x22B5, 0x0338)) + table.put(0xF918, intArrayOf(0x843D)) + table.put(0x22EC, intArrayOf(0x22B4, 0x0338)) + table.put(0xF917, intArrayOf(0x73DE)) + table.put(0x22EB, intArrayOf(0x22B3, 0x0338)) + table.put(0xF916, intArrayOf(0x70D9)) + table.put(0x22EA, intArrayOf(0x22B2, 0x0338)) + table.put(0xF915, intArrayOf(0x6D1B)) + table.put(0xF914, intArrayOf(0x6A02)) + table.put(0xF913, intArrayOf(0x908F)) + table.put(0xF912, intArrayOf(0x88F8)) + table.put(0xF911, intArrayOf(0x87BA)) + table.put(0xF910, intArrayOf(0x863F)) + table.put(0x22E3, intArrayOf(0x2292, 0x0338)) + table.put(0xF90F, intArrayOf(0x7F85)) + table.put(0x22E2, intArrayOf(0x2291, 0x0338)) + table.put(0xF90E, intArrayOf(0x7669)) + table.put(0x22E1, intArrayOf(0x227D, 0x0338)) + table.put(0xF90D, intArrayOf(0x61F6)) + table.put(0x22E0, intArrayOf(0x227C, 0x0338)) + table.put(0xF90C, intArrayOf(0x5948)) + table.put(0xF90B, intArrayOf(0x5587)) + table.put(0xF90A, intArrayOf(0x91D1)) + table.put(0xF909, intArrayOf(0x5951)) + table.put(0xF908, intArrayOf(0x9F9C)) + table.put(0xF907, intArrayOf(0x9F9C)) + table.put(0xF906, intArrayOf(0x53E5)) + table.put(0xF905, intArrayOf(0x4E32)) + table.put(0xF904, intArrayOf(0x6ED1)) + table.put(0xF903, intArrayOf(0x8CC8)) + table.put(0xF902, intArrayOf(0x8ECA)) + table.put(0xF901, intArrayOf(0x66F4)) + table.put(0xF900, intArrayOf(0x8C48)) + table.put(0x00FF, intArrayOf(0x0079, 0x0308)) + table.put(0x00FD, intArrayOf(0x0079, 0x0301)) + table.put(0x00FC, intArrayOf(0x0075, 0x0308)) + table.put(0x00FB, intArrayOf(0x0075, 0x0302)) + table.put(0x00FA, intArrayOf(0x0075, 0x0301)) + table.put(0x00F9, intArrayOf(0x0075, 0x0300)) + table.put(0x00F6, intArrayOf(0x006F, 0x0308)) + table.put(0x00F5, intArrayOf(0x006F, 0x0303)) + table.put(0x00F4, intArrayOf(0x006F, 0x0302)) + table.put(0x00F3, intArrayOf(0x006F, 0x0301)) + table.put(0x00F2, intArrayOf(0x006F, 0x0300)) + table.put(0x00F1, intArrayOf(0x006E, 0x0303)) + table.put(0x00EF, intArrayOf(0x0069, 0x0308)) + table.put(0x00EE, intArrayOf(0x0069, 0x0302)) + table.put(0x00ED, intArrayOf(0x0069, 0x0301)) + table.put(0x00EC, intArrayOf(0x0069, 0x0300)) + table.put(0x00EB, intArrayOf(0x0065, 0x0308)) + table.put(0x00EA, intArrayOf(0x0065, 0x0302)) + table.put(0x00E9, intArrayOf(0x0065, 0x0301)) + table.put(0x00E8, intArrayOf(0x0065, 0x0300)) + table.put(0x00E7, intArrayOf(0x0063, 0x0327)) + table.put(0x00E5, intArrayOf(0x0061, 0x030A)) + table.put(0x00E4, intArrayOf(0x0061, 0x0308)) + table.put(0x00E3, intArrayOf(0x0061, 0x0303)) + table.put(0x00E2, intArrayOf(0x0061, 0x0302)) + table.put(0x00E1, intArrayOf(0x0061, 0x0301)) + table.put(0x00E0, intArrayOf(0x0061, 0x0300)) + table.put(0x00DD, intArrayOf(0x0059, 0x0301)) + table.put(0x00DC, intArrayOf(0x0055, 0x0308)) + table.put(0x00DB, intArrayOf(0x0055, 0x0302)) + table.put(0x00DA, intArrayOf(0x0055, 0x0301)) + table.put(0x22AF, intArrayOf(0x22AB, 0x0338)) + table.put(0x22AE, intArrayOf(0x22A9, 0x0338)) + table.put(0x22AD, intArrayOf(0x22A8, 0x0338)) + table.put(0x22AC, intArrayOf(0x22A2, 0x0338)) + table.put(0x00D9, intArrayOf(0x0055, 0x0300)) + table.put(0x00D6, intArrayOf(0x004F, 0x0308)) + table.put(0x00D5, intArrayOf(0x004F, 0x0303)) + table.put(0x00D4, intArrayOf(0x004F, 0x0302)) + table.put(0x00D3, intArrayOf(0x004F, 0x0301)) + table.put(0x00D2, intArrayOf(0x004F, 0x0300)) + table.put(0x00D1, intArrayOf(0x004E, 0x0303)) + table.put(0x309E, intArrayOf(0x309D, 0x3099)) + table.put(0x00CF, intArrayOf(0x0049, 0x0308)) + table.put(0x00CE, intArrayOf(0x0049, 0x0302)) + table.put(0x00CD, intArrayOf(0x0049, 0x0301)) + table.put(0x00CC, intArrayOf(0x0049, 0x0300)) + table.put(0x00CB, intArrayOf(0x0045, 0x0308)) + table.put(0x00CA, intArrayOf(0x0045, 0x0302)) + table.put(0x00C9, intArrayOf(0x0045, 0x0301)) + table.put(0x3094, intArrayOf(0x3046, 0x3099)) + table.put(0x00C8, intArrayOf(0x0045, 0x0300)) + table.put(0x00C7, intArrayOf(0x0043, 0x0327)) + table.put(0x00C5, intArrayOf(0x0041, 0x030A)) + table.put(0x00C4, intArrayOf(0x0041, 0x0308)) + table.put(0x00C3, intArrayOf(0x0041, 0x0303)) + table.put(0x00C2, intArrayOf(0x0041, 0x0302)) + table.put(0x00C1, intArrayOf(0x0041, 0x0301)) + table.put(0x00C0, intArrayOf(0x0041, 0x0300)) + table.put(0x307D, intArrayOf(0x307B, 0x309A)) + table.put(0x307C, intArrayOf(0x307B, 0x3099)) + table.put(0x2F89F, intArrayOf(0x5FF9)) + table.put(0x307A, intArrayOf(0x3078, 0x309A)) + table.put(0x2F89E, intArrayOf(0x5FD7)) + table.put(0x2F89D, intArrayOf(0x5FCD)) + table.put(0x2F89C, intArrayOf(0x5F9A)) + table.put(0x2F89B, intArrayOf(0x38E3)) + table.put(0x2F89A, intArrayOf(0x5F6B)) + table.put(0x3079, intArrayOf(0x3078, 0x3099)) + table.put(0x3077, intArrayOf(0x3075, 0x309A)) + table.put(0x3076, intArrayOf(0x3075, 0x3099)) + table.put(0x2F899, intArrayOf(0x5F62)) + table.put(0x3074, intArrayOf(0x3072, 0x309A)) + table.put(0x2F898, intArrayOf(0x261DA)) + table.put(0x3073, intArrayOf(0x3072, 0x3099)) + table.put(0x2F897, intArrayOf(0x232B8)) + table.put(0x2F896, intArrayOf(0x38C7)) + table.put(0x3071, intArrayOf(0x306F, 0x309A)) + table.put(0x2F895, intArrayOf(0x5F22)) + table.put(0x3070, intArrayOf(0x306F, 0x3099)) + table.put(0x2F894, intArrayOf(0x5F22)) + table.put(0x2F893, intArrayOf(0x8201)) + table.put(0x2F892, intArrayOf(0x22331)) + table.put(0x2F891, intArrayOf(0x22331)) + table.put(0x2F890, intArrayOf(0x5EFE)) + table.put(0x2F88F, intArrayOf(0x2A392)) + table.put(0x2F88E, intArrayOf(0x5ECA)) + table.put(0x2F88D, intArrayOf(0x5EB6)) + table.put(0x2F88C, intArrayOf(0x5EB3)) + table.put(0x2F88B, intArrayOf(0x5EB0)) + table.put(0x2F88A, intArrayOf(0x387C)) + table.put(0x3069, intArrayOf(0x3068, 0x3099)) + table.put(0x3067, intArrayOf(0x3066, 0x3099)) + table.put(0x3065, intArrayOf(0x3064, 0x3099)) + table.put(0x2F889, intArrayOf(0x22183)) + table.put(0x2F888, intArrayOf(0x3862)) + table.put(0x2F887, intArrayOf(0x5E69)) + table.put(0x3062, intArrayOf(0x3061, 0x3099)) + table.put(0x2F886, intArrayOf(0x5E3D)) + table.put(0x2F885, intArrayOf(0x5E28)) + table.put(0x3060, intArrayOf(0x305F, 0x3099)) + table.put(0x2F884, intArrayOf(0x5DFD)) + table.put(0x2F883, intArrayOf(0x382F)) + table.put(0x2F882, intArrayOf(0x5DE2)) + table.put(0x2F881, intArrayOf(0x5DE1)) + table.put(0x2F880, intArrayOf(0x5D7C)) + table.put(0x305E, intArrayOf(0x305D, 0x3099)) + table.put(0x305C, intArrayOf(0x305B, 0x3099)) + table.put(0x2F87F, intArrayOf(0x5D6B)) + table.put(0x305A, intArrayOf(0x3059, 0x3099)) + table.put(0x2F87E, intArrayOf(0x5D6E)) + table.put(0x2F87D, intArrayOf(0x21DE6)) + table.put(0x2F87C, intArrayOf(0x5D43)) + table.put(0x2F87B, intArrayOf(0x21DE4)) + table.put(0x2F87A, intArrayOf(0x5C8D)) + table.put(0x3058, intArrayOf(0x3057, 0x3099)) + table.put(0x3056, intArrayOf(0x3055, 0x3099)) + table.put(0x2F879, intArrayOf(0x5CC0)) + table.put(0x3054, intArrayOf(0x3053, 0x3099)) + table.put(0x2F878, intArrayOf(0x5C6E)) + table.put(0x2F877, intArrayOf(0x5C60)) + table.put(0x3052, intArrayOf(0x3051, 0x3099)) + table.put(0x2F876, intArrayOf(0x3781)) + table.put(0x2F875, intArrayOf(0x5C22)) + table.put(0x3050, intArrayOf(0x304F, 0x3099)) + table.put(0x2F874, intArrayOf(0x5F53)) + table.put(0x2F873, intArrayOf(0x5C06)) + table.put(0x2F872, intArrayOf(0x5BFF)) + table.put(0x2F871, intArrayOf(0x21B18)) + table.put(0x2F870, intArrayOf(0x5BF3)) + table.put(0x304E, intArrayOf(0x304D, 0x3099)) + table.put(0x304C, intArrayOf(0x304B, 0x3099)) + table.put(0x2F86F, intArrayOf(0x5BE7)) + table.put(0x2F86E, intArrayOf(0x5BD8)) + table.put(0x2F86D, intArrayOf(0x5BC3)) + table.put(0x2F86C, intArrayOf(0x219C8)) + table.put(0x2F86B, intArrayOf(0x5B3E)) + table.put(0x2F86A, intArrayOf(0x5B3E)) + table.put(0x2F869, intArrayOf(0x5B08)) + table.put(0x2F868, intArrayOf(0x36FC)) + table.put(0x2F867, intArrayOf(0x36EE)) + table.put(0x0CCB, intArrayOf(0x0CC6, 0x0CC2, 0x0CD5)) + table.put(0x2F866, intArrayOf(0x5A66)) + table.put(0x0CCA, intArrayOf(0x0CC6, 0x0CC2)) + table.put(0x2F865, intArrayOf(0x59D8)) + table.put(0x2F864, intArrayOf(0x5A27)) + table.put(0x2F863, intArrayOf(0x5A1B)) + table.put(0x03D4, intArrayOf(0x03D2, 0x0308)) + table.put(0x2F862, intArrayOf(0x59EC)) + table.put(0x03D3, intArrayOf(0x03D2, 0x0301)) + table.put(0x2F861, intArrayOf(0x216EA)) + table.put(0x2F860, intArrayOf(0x216A8)) + table.put(0x0CC8, intArrayOf(0x0CC6, 0x0CD6)) + table.put(0x0CC7, intArrayOf(0x0CC6, 0x0CD5)) + table.put(0x2F85F, intArrayOf(0x5962)) + table.put(0x2F85E, intArrayOf(0x5922)) + table.put(0x2F85D, intArrayOf(0x591A)) + table.put(0x0CC0, intArrayOf(0x0CBF, 0x0CD5)) + table.put(0x03CE, intArrayOf(0x03C9, 0x0301)) + table.put(0x2F85C, intArrayOf(0x5906)) + table.put(0x03CD, intArrayOf(0x03C5, 0x0301)) + table.put(0x2F85B, intArrayOf(0x58F7)) + table.put(0x03CC, intArrayOf(0x03BF, 0x0301)) + table.put(0x2F85A, intArrayOf(0x58F2)) + table.put(0x03CB, intArrayOf(0x03C5, 0x0308)) + table.put(0x03CA, intArrayOf(0x03B9, 0x0308)) + table.put(0x2F859, intArrayOf(0x214E4)) + table.put(0x2F858, intArrayOf(0x58AC)) + table.put(0x2F857, intArrayOf(0x5831)) + table.put(0x2F856, intArrayOf(0x5832)) + table.put(0x2F855, intArrayOf(0x578B)) + table.put(0x2F854, intArrayOf(0x580D)) + table.put(0x2F853, intArrayOf(0x57F4)) + table.put(0x2F852, intArrayOf(0x57CE)) + table.put(0x2F851, intArrayOf(0x58EE)) + table.put(0x2F850, intArrayOf(0x5207)) + table.put(0x2F84F, intArrayOf(0x5674)) + table.put(0x2F84E, intArrayOf(0x5651)) + table.put(0x2F84D, intArrayOf(0x5717)) + table.put(0x2F84C, intArrayOf(0x5606)) + table.put(0x2F84B, intArrayOf(0x5716)) + table.put(0x2F84A, intArrayOf(0x55C2)) + table.put(0x2F849, intArrayOf(0x55B3)) + table.put(0x2F848, intArrayOf(0x55AB)) + table.put(0x2F847, intArrayOf(0x5599)) + table.put(0x2F846, intArrayOf(0x5584)) + table.put(0x2F845, intArrayOf(0x5584)) + table.put(0x2F844, intArrayOf(0x5563)) + table.put(0x2F843, intArrayOf(0x5553)) + table.put(0x2F842, intArrayOf(0x5510)) + table.put(0x2F841, intArrayOf(0x54F6)) + table.put(0x2F840, intArrayOf(0x54A2)) + table.put(0x03B0, intArrayOf(0x03C5, 0x0308, 0x0301)) + table.put(0x2F83F, intArrayOf(0x5468)) + table.put(0x2F83E, intArrayOf(0x5448)) + table.put(0x03AF, intArrayOf(0x03B9, 0x0301)) + table.put(0x2F83D, intArrayOf(0x5438)) + table.put(0x03AE, intArrayOf(0x03B7, 0x0301)) + table.put(0x2F83C, intArrayOf(0x549E)) + table.put(0x03AD, intArrayOf(0x03B5, 0x0301)) + table.put(0x2F83B, intArrayOf(0x5406)) + table.put(0x03AC, intArrayOf(0x03B1, 0x0301)) + table.put(0x2F83A, intArrayOf(0x53F1)) + table.put(0x03AB, intArrayOf(0x03A5, 0x0308)) + table.put(0x03AA, intArrayOf(0x0399, 0x0308)) + table.put(0x2F839, intArrayOf(0x53EB)) + table.put(0x2F838, intArrayOf(0x20B63)) + table.put(0x2F837, intArrayOf(0x53DF)) + table.put(0x2F836, intArrayOf(0x53CA)) + table.put(0x2F835, intArrayOf(0x7070)) + table.put(0x2F834, intArrayOf(0x20A2C)) + table.put(0x2F833, intArrayOf(0x537F)) + table.put(0x2F832, intArrayOf(0x537F)) + table.put(0x2F831, intArrayOf(0x537F)) + table.put(0x2289, intArrayOf(0x2287, 0x0338)) + table.put(0x2F830, intArrayOf(0x537D)) + table.put(0x2288, intArrayOf(0x2286, 0x0338)) + table.put(0x2285, intArrayOf(0x2283, 0x0338)) + table.put(0x2284, intArrayOf(0x2282, 0x0338)) + table.put(0x2281, intArrayOf(0x227B, 0x0338)) + table.put(0x2280, intArrayOf(0x227A, 0x0338)) + table.put(0x2F82F, intArrayOf(0x5373)) + table.put(0x2F82E, intArrayOf(0x535A)) + table.put(0x2F82D, intArrayOf(0x5351)) + table.put(0x2F82C, intArrayOf(0x5349)) + table.put(0x2F82B, intArrayOf(0x5317)) + table.put(0x2F82A, intArrayOf(0x5306)) + table.put(0x2F829, intArrayOf(0x5305)) + table.put(0x2F828, intArrayOf(0x52FA)) + table.put(0x2F827, intArrayOf(0x52E4)) + table.put(0x2F826, intArrayOf(0x52C9)) + table.put(0x2F825, intArrayOf(0x52C7)) + table.put(0x2F824, intArrayOf(0x3515)) + table.put(0x2F823, intArrayOf(0x5277)) + table.put(0x2F822, intArrayOf(0x5272)) + table.put(0x2F821, intArrayOf(0x5246)) + table.put(0x2279, intArrayOf(0x2277, 0x0338)) + table.put(0x2F820, intArrayOf(0x523B)) + table.put(0x2278, intArrayOf(0x2276, 0x0338)) + table.put(0x2275, intArrayOf(0x2273, 0x0338)) + table.put(0x2274, intArrayOf(0x2272, 0x0338)) + table.put(0x2271, intArrayOf(0x2265, 0x0338)) + table.put(0x2270, intArrayOf(0x2264, 0x0338)) + table.put(0x2F81F, intArrayOf(0x34DF)) + table.put(0x2F81E, intArrayOf(0x5203)) + table.put(0x2F81D, intArrayOf(0x51F5)) + table.put(0x2F81C, intArrayOf(0x291DF)) + table.put(0x2F81B, intArrayOf(0x51B5)) + table.put(0xFB4E, intArrayOf(0x05E4, 0x05BF)) + table.put(0x2F81A, intArrayOf(0x51AC)) + table.put(0xFB4D, intArrayOf(0x05DB, 0x05BF)) + table.put(0xFB4C, intArrayOf(0x05D1, 0x05BF)) + table.put(0xFB4B, intArrayOf(0x05D5, 0x05B9)) + table.put(0x226F, intArrayOf(0x003E, 0x0338)) + table.put(0xFB4A, intArrayOf(0x05EA, 0x05BC)) + table.put(0x226E, intArrayOf(0x003C, 0x0338)) + table.put(0x226D, intArrayOf(0x224D, 0x0338)) + table.put(0x2F819, intArrayOf(0x4ECC)) + table.put(0x2F818, intArrayOf(0x51A4)) + table.put(0x2F817, intArrayOf(0x5197)) + table.put(0x2F816, intArrayOf(0x2054B)) + table.put(0xFB49, intArrayOf(0x05E9, 0x05BC)) + table.put(0x2F815, intArrayOf(0x518D)) + table.put(0xFB48, intArrayOf(0x05E8, 0x05BC)) + table.put(0x2F814, intArrayOf(0x5167)) + table.put(0xFB47, intArrayOf(0x05E7, 0x05BC)) + table.put(0x2F813, intArrayOf(0x34B9)) + table.put(0xFB46, intArrayOf(0x05E6, 0x05BC)) + table.put(0x2F812, intArrayOf(0x2051C)) + table.put(0x2F811, intArrayOf(0x5177)) + table.put(0xFB44, intArrayOf(0x05E4, 0x05BC)) + table.put(0x2F810, intArrayOf(0x5164)) + table.put(0xFB43, intArrayOf(0x05E3, 0x05BC)) + table.put(0xFB41, intArrayOf(0x05E1, 0x05BC)) + table.put(0xFB40, intArrayOf(0x05E0, 0x05BC)) + table.put(0x2262, intArrayOf(0x2261, 0x0338)) + table.put(0x2260, intArrayOf(0x003D, 0x0338)) + table.put(0x2F80F, intArrayOf(0x5154)) + table.put(0x2F80E, intArrayOf(0x514D)) + table.put(0x2F80D, intArrayOf(0x2063A)) + table.put(0x2F80C, intArrayOf(0x349E)) + table.put(0x2F80B, intArrayOf(0x50CF)) + table.put(0xFB3E, intArrayOf(0x05DE, 0x05BC)) + table.put(0x2F80A, intArrayOf(0x50E7)) + table.put(0xFB3C, intArrayOf(0x05DC, 0x05BC)) + table.put(0xFB3B, intArrayOf(0x05DB, 0x05BC)) + table.put(0xFB3A, intArrayOf(0x05DA, 0x05BC)) + table.put(0x2F809, intArrayOf(0x5099)) + table.put(0x2F808, intArrayOf(0x507A)) + table.put(0x2F807, intArrayOf(0x5002)) + table.put(0x2F806, intArrayOf(0x4FBB)) + table.put(0xFB39, intArrayOf(0x05D9, 0x05BC)) + table.put(0x2F805, intArrayOf(0x4FAE)) + table.put(0xFB38, intArrayOf(0x05D8, 0x05BC)) + table.put(0x2F804, intArrayOf(0x4F60)) + table.put(0x2F803, intArrayOf(0x20122)) + table.put(0xFB36, intArrayOf(0x05D6, 0x05BC)) + table.put(0x2F802, intArrayOf(0x4E41)) + table.put(0xFB35, intArrayOf(0x05D5, 0x05BC)) + table.put(0x2F801, intArrayOf(0x4E38)) + table.put(0xFB34, intArrayOf(0x05D4, 0x05BC)) + table.put(0x2F800, intArrayOf(0x4E3D)) + table.put(0x06D3, intArrayOf(0x06D2, 0x0654)) + table.put(0xFB33, intArrayOf(0x05D3, 0x05BC)) + table.put(0xFB32, intArrayOf(0x05D2, 0x05BC)) + table.put(0xFB31, intArrayOf(0x05D1, 0x05BC)) + table.put(0xFB30, intArrayOf(0x05D0, 0x05BC)) + table.put(0xFB2F, intArrayOf(0x05D0, 0x05B8)) + table.put(0xFB2E, intArrayOf(0x05D0, 0x05B7)) + table.put(0xFB2D, intArrayOf(0x05E9, 0x05BC, 0x05C2)) + table.put(0xFB2C, intArrayOf(0x05E9, 0x05BC, 0x05C1)) + table.put(0xFB2B, intArrayOf(0x05E9, 0x05C2)) + table.put(0xFB2A, intArrayOf(0x05E9, 0x05C1)) + table.put(0x2249, intArrayOf(0x2248, 0x0338)) + table.put(0x2247, intArrayOf(0x2245, 0x0338)) + table.put(0x06C2, intArrayOf(0x06C1, 0x0654)) + table.put(0x06C0, intArrayOf(0x06D5, 0x0654)) + table.put(0x2244, intArrayOf(0x2243, 0x0338)) + table.put(0x0FB9, intArrayOf(0x0F90, 0x0FB5)) + table.put(0x2241, intArrayOf(0x223C, 0x0338)) + table.put(0xFB1F, intArrayOf(0x05F2, 0x05B7)) + table.put(0xFB1D, intArrayOf(0x05D9, 0x05B4)) + table.put(0x0FAC, intArrayOf(0x0FAB, 0x0FB7)) + table.put(0x0FA7, intArrayOf(0x0FA6, 0x0FB7)) + table.put(0x0FA2, intArrayOf(0x0FA1, 0x0FB7)) + table.put(0x2226, intArrayOf(0x2225, 0x0338)) + table.put(0x2224, intArrayOf(0x2223, 0x0338)) + table.put(0x220C, intArrayOf(0x220B, 0x0338)) + table.put(0x2209, intArrayOf(0x2208, 0x0338)) + table.put(0x0390, intArrayOf(0x03B9, 0x0308, 0x0301)) + table.put(0x2204, intArrayOf(0x2203, 0x0338)) + table.put(0x038F, intArrayOf(0x03A9, 0x0301)) + table.put(0x038E, intArrayOf(0x03A5, 0x0301)) + table.put(0x038C, intArrayOf(0x039F, 0x0301)) + table.put(0x09DF, intArrayOf(0x09AF, 0x09BC)) + table.put(0x038A, intArrayOf(0x0399, 0x0301)) + table.put(0x09DD, intArrayOf(0x09A2, 0x09BC)) + table.put(0x09DC, intArrayOf(0x09A1, 0x09BC)) + table.put(0x0389, intArrayOf(0x0397, 0x0301)) + table.put(0x0388, intArrayOf(0x0395, 0x0301)) + table.put(0x0387, intArrayOf(0x00B7)) + table.put(0x0386, intArrayOf(0x0391, 0x0301)) + table.put(0x0385, intArrayOf(0x00A8, 0x0301)) + table.put(0x037E, intArrayOf(0x003B)) + table.put(0x09CC, intArrayOf(0x09C7, 0x09D7)) + table.put(0x09CB, intArrayOf(0x09C7, 0x09BE)) + table.put(0x0374, intArrayOf(0x02B9)) + table.put(0x0C48, intArrayOf(0x0C46, 0x0C56)) + table.put(0x0F9D, intArrayOf(0x0F9C, 0x0FB7)) + table.put(0x0344, intArrayOf(0x0308, 0x0301)) + table.put(0x0343, intArrayOf(0x0313)) + table.put(0x0341, intArrayOf(0x0301)) + table.put(0x0340, intArrayOf(0x0300)) + table.put(0x0F93, intArrayOf(0x0F92, 0x0FB7)) + table.put(0x0F81, intArrayOf(0x0F71, 0x0F80)) + table.put(0x0F78, intArrayOf(0x0FB3, 0x0F80)) + table.put(0x1D1C0, intArrayOf(0x1D1BA, 0x1D165, 0x1D16F)) + table.put(0x0F76, intArrayOf(0x0FB2, 0x0F80)) + table.put(0x0F75, intArrayOf(0x0F71, 0x0F74)) + table.put(0x0F73, intArrayOf(0x0F71, 0x0F72)) + table.put(0x1D1BF, intArrayOf(0x1D1B9, 0x1D165, 0x1D16F)) + table.put(0x1D1BE, intArrayOf(0x1D1BA, 0x1D165, 0x1D16E)) + table.put(0x1D1BD, intArrayOf(0x1D1B9, 0x1D165, 0x1D16E)) + table.put(0x1D1BC, intArrayOf(0x1D1BA, 0x1D165)) + table.put(0x1D1BB, intArrayOf(0x1D1B9, 0x1D165)) + table.put(0x0F69, intArrayOf(0x0F40, 0x0FB5)) + table.put(0x0F5C, intArrayOf(0x0F5B, 0x0FB7)) + table.put(0xFAD9, intArrayOf(0x9F8E)) + table.put(0xFAD8, intArrayOf(0x9F43)) + table.put(0xFAD7, intArrayOf(0x27ED3)) + table.put(0xFAD6, intArrayOf(0x25CD0)) + table.put(0xFAD5, intArrayOf(0x25249)) + table.put(0xFAD4, intArrayOf(0x4039)) + table.put(0xFAD3, intArrayOf(0x4018)) + table.put(0xFAD2, intArrayOf(0x3B9D)) + table.put(0xFAD1, intArrayOf(0x233D5)) + table.put(0xFAD0, intArrayOf(0x22844)) + table.put(0x0F57, intArrayOf(0x0F56, 0x0FB7)) + table.put(0x0F52, intArrayOf(0x0F51, 0x0FB7)) + table.put(0xFACF, intArrayOf(0x2284A)) + table.put(0xFACE, intArrayOf(0x9F9C)) + table.put(0xFACD, intArrayOf(0x9B12)) + table.put(0xFACC, intArrayOf(0x983B)) + table.put(0xFACB, intArrayOf(0x980B)) + table.put(0xFACA, intArrayOf(0x97FF)) + table.put(0x0F4D, intArrayOf(0x0F4C, 0x0FB7)) + table.put(0xFAC9, intArrayOf(0x97DB)) + table.put(0xFAC8, intArrayOf(0x9756)) + table.put(0xFAC7, intArrayOf(0x96E3)) + table.put(0xFAC6, intArrayOf(0x967C)) + table.put(0xFAC5, intArrayOf(0x9276)) + table.put(0x2ADC, intArrayOf(0x2ADD, 0x0338)) + table.put(0xFAC4, intArrayOf(0x9199)) + table.put(0xFAC3, intArrayOf(0x9072)) + table.put(0xFAC2, intArrayOf(0x8F38)) + table.put(0xFAC1, intArrayOf(0x8D08)) + table.put(0xFAC0, intArrayOf(0x8B8A)) + table.put(0x0F43, intArrayOf(0x0F42, 0x0FB7)) + table.put(0xFABF, intArrayOf(0x8B39)) + table.put(0xFABE, intArrayOf(0x8AED)) + table.put(0xFABD, intArrayOf(0x8AFE)) + table.put(0xFABC, intArrayOf(0x8B01)) + table.put(0xFABB, intArrayOf(0x8ACB)) + table.put(0xFABA, intArrayOf(0x8AF8)) + table.put(0xFAB9, intArrayOf(0x8ABF)) + table.put(0xFAB8, intArrayOf(0x8996)) + table.put(0xFAB7, intArrayOf(0x8986)) + table.put(0xFAB6, intArrayOf(0x8941)) + table.put(0xFAB5, intArrayOf(0x8779)) + table.put(0xFAB4, intArrayOf(0x83EF)) + table.put(0xFAB3, intArrayOf(0x8352)) + table.put(0xFAB2, intArrayOf(0x8005)) + table.put(0xFAB1, intArrayOf(0x7F3E)) + table.put(0xFAB0, intArrayOf(0x7DF4)) + table.put(0xFAAF, intArrayOf(0x7D5B)) + table.put(0xFAAE, intArrayOf(0x7C7B)) + table.put(0xFAAD, intArrayOf(0x7BC0)) + table.put(0xFAAC, intArrayOf(0x7AB1)) + table.put(0xFAAB, intArrayOf(0x78CC)) + table.put(0x2FA1D, intArrayOf(0x2A600)) + table.put(0x21CF, intArrayOf(0x21D2, 0x0338)) + table.put(0xFAAA, intArrayOf(0x7740)) + table.put(0x2FA1C, intArrayOf(0x9F3B)) + table.put(0x21CE, intArrayOf(0x21D4, 0x0338)) + table.put(0x2FA1B, intArrayOf(0x9F16)) + table.put(0x21CD, intArrayOf(0x21D0, 0x0338)) + table.put(0x2FA1A, intArrayOf(0x9F0F)) + table.put(0xFAA9, intArrayOf(0x774A)) + table.put(0xFAA8, intArrayOf(0x76F4)) + table.put(0xFAA7, intArrayOf(0x76DB)) + table.put(0x2FA19, intArrayOf(0x9F05)) + table.put(0xFAA6, intArrayOf(0x76CA)) + table.put(0x2FA18, intArrayOf(0x9EFE)) + table.put(0xFAA5, intArrayOf(0x761F)) + table.put(0x2FA17, intArrayOf(0x9EF9)) + table.put(0xFAA4, intArrayOf(0x761D)) + table.put(0x2FA16, intArrayOf(0x4D56)) + table.put(0xFAA3, intArrayOf(0x753B)) + table.put(0x2FA15, intArrayOf(0x9EBB)) + table.put(0xFAA2, intArrayOf(0x7506)) + table.put(0x2FA14, intArrayOf(0x2A291)) + table.put(0xFAA1, intArrayOf(0x7471)) + table.put(0x2FA13, intArrayOf(0x2A20E)) + table.put(0xFAA0, intArrayOf(0x732A)) + table.put(0x2FA12, intArrayOf(0x2A105)) + table.put(0x2FA11, intArrayOf(0x4CF8)) + table.put(0x2FA10, intArrayOf(0x2A0CE)) + table.put(0x2FA0F, intArrayOf(0x9D67)) + table.put(0x2FA0E, intArrayOf(0x4CED)) + table.put(0x2FA0D, intArrayOf(0x4CCE)) + table.put(0x2FA0C, intArrayOf(0x9CFD)) + table.put(0x2FA0B, intArrayOf(0x9C40)) + table.put(0x2FA0A, intArrayOf(0x9B12)) + table.put(0x2FA09, intArrayOf(0x29B30)) + table.put(0x0626, intArrayOf(0x064A, 0x0654)) + table.put(0x2FA08, intArrayOf(0x4BCE)) + table.put(0x0625, intArrayOf(0x0627, 0x0655)) + table.put(0x2FA07, intArrayOf(0x99FE)) + table.put(0x0624, intArrayOf(0x0648, 0x0654)) + table.put(0x2FA06, intArrayOf(0x99C2)) + table.put(0x0623, intArrayOf(0x0627, 0x0654)) + table.put(0x2FA05, intArrayOf(0x99A7)) + table.put(0x0622, intArrayOf(0x0627, 0x0653)) + table.put(0x2FA04, intArrayOf(0x9929)) + table.put(0x2FA03, intArrayOf(0x4B33)) + table.put(0x2FA02, intArrayOf(0x98E2)) + table.put(0x2FA01, intArrayOf(0x295B6)) + table.put(0x2FA00, intArrayOf(0x9829)) + table.put(0x21AE, intArrayOf(0x2194, 0x0338)) + table.put(0x095F, intArrayOf(0x092F, 0x093C)) + table.put(0x095E, intArrayOf(0x092B, 0x093C)) + table.put(0x095D, intArrayOf(0x0922, 0x093C)) + table.put(0x095C, intArrayOf(0x0921, 0x093C)) + table.put(0x095B, intArrayOf(0x091C, 0x093C)) + table.put(0x095A, intArrayOf(0x0917, 0x093C)) + table.put(0x0959, intArrayOf(0x0916, 0x093C)) + table.put(0x0958, intArrayOf(0x0915, 0x093C)) + table.put(0x0934, intArrayOf(0x0933, 0x093C)) + table.put(0x0931, intArrayOf(0x0930, 0x093C)) + return@lazy table + } +} + + +/** + * This class gives us a hashtable that contains information about + * symbols that are one symbol decompositions that is + * generated from + * http://www.unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt. + */ +internal object SingleDecompositions{ + val hashSingleDecompositions: IntHash by lazy { + val table = IntHash(1030) + table.put(0x6688, 0) + table.put(0x00B7, 0) + table.put(0x86A9, 0) + table.put(0x00B4, 0) + table.put(0x7409, 0) + table.put(0x559D, 0) + table.put(0x884C, 0) + table.put(0x523B, 0) + table.put(0x6DCB, 0) + table.put(0x8AFE, 0) + table.put(0x523A, 0) + table.put(0x7406, 0) + table.put(0x53F1, 0) + table.put(0x5599, 0) + table.put(0x6F6E, 0) + table.put(0x8AF8, 0) + table.put(0x3862, 0) + table.put(0x5B28, 0) + table.put(0x53EB, 0) + table.put(0x2300A, 0) + table.put(0x9929, 0) + table.put(0x9928, 0) + table.put(0x6674, 0) + table.put(0x8046, 0) + table.put(0x53E5, 0) + table.put(0x8AED, 0) + table.put(0x2054B, 0) + table.put(0x5229, 0) + table.put(0x64C4, 0) + table.put(0x278AE, 0) + table.put(0x53DF, 0) + table.put(0x5587, 0) + table.put(0x2335F, 0) + table.put(0x5584, 0) + table.put(0x92D8, 0) + table.put(0x92D7, 0) + table.put(0x5E74, 0) + table.put(0x774A, 0) + table.put(0x911B, 0) + table.put(0x81ED, 0) + table.put(0x78FB, 0) + table.put(0x5CC0, 0) + table.put(0x7740, 0) + table.put(0x81E8, 0) + table.put(0x9111, 0) + table.put(0x9D67, 0) + table.put(0x5217, 0) + table.put(0x2A20E, 0) + table.put(0x5B08, 0) + table.put(0x5E69, 0) + table.put(0x8AD6, 0) + table.put(0x267B5, 0) + table.put(0x412F, 0) + table.put(0x53CA, 0) + table.put(0x8AD2, 0) + table.put(0x881F, 0) + table.put(0x53C3, 0) + table.put(0x2940A, 0) + table.put(0x8ACB, 0) + table.put(0x256C5, 0) + table.put(0x4BCE, 0) + table.put(0x81D8, 0) + table.put(0x5207, 0) + table.put(0x267A7, 0) + table.put(0x3AE4, 0) + table.put(0x5203, 0) + table.put(0x5563, 0) + table.put(0x7295, 0) + table.put(0x20525, 0) + table.put(0x6649, 0) + table.put(0x23F8E, 0) + table.put(0x382F, 0) + table.put(0x295B6, 0) + table.put(0x67FA, 0) + table.put(0x8ABF, 0) + table.put(0x2051C, 0) + table.put(0x03CE, 0) + table.put(0x67F3, 0) + table.put(0x9EFE, 0) + table.put(0x03CD, 0) + table.put(0x03CC, 0) + table.put(0x5555, 0) + table.put(0x5553, 0) + table.put(0x771F, 0) + table.put(0x8801, 0) + table.put(0x771E, 0) + table.put(0x446B, 0) + table.put(0x7280, 0) + table.put(0x9EF9, 0) + table.put(0x8005, 0) + table.put(0x70D9, 0) + table.put(0x8363, 0) + table.put(0x8001, 0) + table.put(0x5FF9, 0) + table.put(0x6F23, 0) + table.put(0x6F22, 0) + table.put(0x8C55, 0) + table.put(0x5E3D, 0) + table.put(0x5FF5, 0) + table.put(0x8AAA, 0) + table.put(0x78CC, 0) + table.put(0x78CA, 0) + table.put(0x4D56, 0) + table.put(0x6190, 0) + table.put(0x8AA0, 0) + table.put(0x618E, 0) + table.put(0x03B9, 0) + table.put(0x56F9, 0) + table.put(0x273CA, 0) + table.put(0x70C8, 0) + table.put(0x8353, 0) + table.put(0x8C48, 0) + table.put(0x8352, 0) + table.put(0x03B0, 0) + table.put(0x25044, 0) + table.put(0x7701, 0) + table.put(0x03AF, 0) + table.put(0x03AE, 0) + table.put(0x6F0F, 0) + table.put(0x03AD, 0) + table.put(0x03AC, 0) + table.put(0x5E28, 0) + table.put(0x8F9E, 0) + table.put(0x3009, 0) + table.put(0x7262, 0) + table.put(0x3008, 0) + table.put(0x23F5E, 0) + table.put(0x8DF0, 0) + table.put(0x03A9, 0) + table.put(0x42A0, 0) + table.put(0x5099, 0) + table.put(0x262D9, 0) + table.put(0x617A, 0) + table.put(0x6613, 0) + table.put(0x84FC, 0) + table.put(0x8DEF, 0) + table.put(0x5FD7, 0) + table.put(0x26D6B, 0) + table.put(0x27CA8, 0) + table.put(0x9ECE, 0) + table.put(0x22B0C, 0) + table.put(0x3C4E, 0) + table.put(0x84F3, 0) + table.put(0x25C80, 0) + table.put(0x45F9, 0) + table.put(0x84F1, 0) + table.put(0x70AD, 0) + table.put(0x5FCD, 0) + table.put(0x7250, 0) + table.put(0x4EE4, 0) + table.put(0x8336, 0) + table.put(0x84EE, 0) + table.put(0x6168, 0) + table.put(0x9415, 0) + table.put(0x24735, 0) + table.put(0x6160, 0) + table.put(0x9EBB, 0) + table.put(0x507A, 0) + table.put(0x5510, 0) + table.put(0x8688, 0) + table.put(0x8323, 0) + table.put(0x7CE8, 0) + table.put(0x8DCB, 0) + table.put(0x7CE7, 0) + table.put(0x7CE3, 0) + table.put(0x4ECC, 0) + table.put(0x831D, 0) + table.put(0x7235, 0) + table.put(0x45D7, 0) + table.put(0x95B7, 0) + table.put(0x5FAD, 0) + table.put(0x614E, 0) + table.put(0x614C, 0) + table.put(0x7B20, 0) + table.put(0x6942, 0) + table.put(0x4EC0, 0) + table.put(0x585E, 0) + table.put(0x5FA9, 0) + table.put(0x8DBC, 0) + table.put(0x585A, 0) + table.put(0x722B, 0) + table.put(0x6148, 0) + table.put(0x8F62, 0) + table.put(0x23A8D, 0) + table.put(0x7CD6, 0) + table.put(0x9756, 0) + table.put(0x95AD, 0) + table.put(0x62FE, 0) + table.put(0x6144, 0) + table.put(0x62FC, 0) + table.put(0x7CD2, 0) + table.put(0x7228, 0) + table.put(0x73DE, 0) + table.put(0x25AA7, 0) + table.put(0x440B, 0) + table.put(0x8669, 0) + table.put(0x34DF, 0) + table.put(0x649D, 0) + table.put(0x8667, 0) + table.put(0x3C18, 0) + table.put(0x649A, 0) + table.put(0x9748, 0) + table.put(0x721B, 0) + table.put(0x98FC, 0) + table.put(0x4EAE, 0) + table.put(0x6D85, 0) + table.put(0x865C, 0) + table.put(0x5840, 0) + table.put(0x20804, 0) + table.put(0x90FD, 0) + table.put(0x7210, 0) + table.put(0x7CBE, 0) + table.put(0x7570, 0) + table.put(0x24263, 0) + table.put(0x0060, 0) + table.put(0x8650, 0) + table.put(0x29B30, 0) + table.put(0x98EF, 0) + table.put(0x90F1, 0) + table.put(0x6D78, 0) + table.put(0x9732, 0) + table.put(0x6D77, 0) + table.put(0x51F5, 0) + table.put(0x6A13, 0) + table.put(0x2A392, 0) + table.put(0x59EC, 0) + table.put(0x5832, 0) + table.put(0x7565, 0) + table.put(0x5831, 0) + table.put(0x5C8D, 0) + table.put(0x8F3B, 0) + table.put(0x647E, 0) + table.put(0x2569A, 0) + table.put(0x62D4, 0) + table.put(0x62D3, 0) + table.put(0x28BFA, 0) + table.put(0x6914, 0) + table.put(0x98E2, 0) + table.put(0x34BB, 0) + table.put(0x8F38, 0) + table.put(0x6D6A, 0) + table.put(0x73B2, 0) + table.put(0x34B9, 0) + table.put(0x62CF, 0) + table.put(0x6477, 0) + table.put(0x226D4, 0) + table.put(0x25F86, 0) + table.put(0x6D69, 0) + table.put(0x9723, 0) + table.put(0x863F, 0) + table.put(0x004B, 0) + table.put(0x7559, 0) + table.put(0x502B, 0) + table.put(0x6A02, 0) + table.put(0x6BBB, 0) + table.put(0x6BBA, 0) + table.put(0x62C9, 0) + table.put(0x8F2A, 0) + table.put(0x7E45, 0) + table.put(0x7E41, 0) + table.put(0x7FFA, 0) + table.put(0x51DE, 0) + table.put(0x59D8, 0) + table.put(0x8996, 0) + table.put(0x51DC, 0) + table.put(0x8F26, 0) + table.put(0x73A5, 0) + table.put(0x6108, 0) + table.put(0x6469, 0) + table.put(0x927C, 0) + table.put(0x9B6F, 0) + table.put(0x537F, 0) + table.put(0x6BAE, 0) + table.put(0x003B, 0) + table.put(0x862D, 0) + table.put(0x537D, 0) + table.put(0x2921A, 0) + table.put(0x898B, 0) + table.put(0x87E1, 0) + table.put(0x5C6E, 0) + table.put(0x90CE, 0) + table.put(0x9276, 0) + table.put(0x7E37, 0) + table.put(0x62B1, 0) + table.put(0x8986, 0) + table.put(0x5375, 0) + table.put(0x51CC, 0) + table.put(0x5373, 0) + table.put(0x580D, 0) + table.put(0x0390, 0) + table.put(0x753E, 0) + table.put(0x5C65, 0) + table.put(0x5C64, 0) + table.put(0x76F4, 0) + table.put(0x21D0B, 0) + table.put(0x40E3, 0) + table.put(0x753B, 0) + table.put(0x5C62, 0) + table.put(0x5C60, 0) + table.put(0x51C9, 0) + table.put(0x038F, 0) + table.put(0x038E, 0) + table.put(0x6452, 0) + table.put(0x038C, 0) + table.put(0x038A, 0) + table.put(0x6D41, 0) + table.put(0x5AB5, 0) + table.put(0x28D77, 0) + table.put(0x0389, 0) + table.put(0x7891, 0) + table.put(0x0388, 0) + table.put(0x0386, 0) + table.put(0x0385, 0) + table.put(0x265A8, 0) + table.put(0x6D3E, 0) + table.put(0x7099, 0) + table.put(0x5002, 0) + table.put(0x8612, 0) + table.put(0x76E7, 0) + table.put(0x788C, 0) + table.put(0x51B7, 0) + table.put(0x2284A, 0) + table.put(0x51B5, 0) + table.put(0x6D34, 0) + table.put(0x535A, 0) + table.put(0x3F1B, 0) + table.put(0x7524, 0) + table.put(0x76DB, 0) + table.put(0x22844, 0) + table.put(0x8606, 0) + table.put(0x9CFD, 0) + table.put(0x51AC, 0) + table.put(0x8964, 0) + table.put(0x87BA, 0) + table.put(0x5351, 0) + table.put(0x5DFD, 0) + table.put(0x6797, 0) + table.put(0x9E9F, 0) + table.put(0x51A4, 0) + table.put(0x6EDB, 0) + table.put(0x5C3F, 0) + table.put(0x707D, 0) + table.put(0x7E09, 0) + table.put(0x76CA, 0) + table.put(0x5F9A, 0) + table.put(0x7FBD, 0) + table.put(0x5349, 0) + table.put(0x9E97, 0) + table.put(0x65E3, 0) + table.put(0x7E02, 0) + table.put(0x65E2, 0) + table.put(0x6D1E, 0) + table.put(0x7077, 0) + table.put(0x3A6C, 0) + table.put(0x6D1B, 0) + table.put(0x6ED1, 0) + table.put(0x93F9, 0) + table.put(0x7070, 0) + table.put(0x786B, 0) + table.put(0x6785, 0) + table.put(0x6422, 0) + table.put(0x533F, 0) + table.put(0x6D16, 0) + table.put(0x54F6, 0) + table.put(0x6ECB, 0) + table.put(0x7506, 0) + table.put(0x9238, 0) + table.put(0x5DE2, 0) + table.put(0x5F8B, 0) + table.put(0x9234, 0) + table.put(0x5DE1, 0) + table.put(0x641C, 0) + table.put(0x677B, 0) + table.put(0x6EC7, 0) + table.put(0x243AB, 0) + table.put(0x2346D, 0) + table.put(0x8941, 0) + table.put(0x4CF8, 0) + table.put(0x5C22, 0) + table.put(0x958B, 0) + table.put(0x9E7F, 0) + table.put(0x6EBA, 0) + table.put(0x65C5, 0) + table.put(0x4CED, 0) + table.put(0x4B33, 0) + table.put(0x813E, 0) + table.put(0x232B8, 0) + table.put(0x9B12, 0) + table.put(0x784E, 0) + table.put(0x25626, 0) + table.put(0x285ED, 0) + table.put(0x704A, 0) + table.put(0x5F6B, 0) + table.put(0x2063A, 0) + table.put(0x2597C, 0) + table.put(0x4E86, 0) + table.put(0x675E, 0) + table.put(0x82E6, 0) + table.put(0x5317, 0) + table.put(0x82E5, 0) + table.put(0x4E82, 0) + table.put(0x5674, 0) + table.put(0x5F69, 0) + table.put(0x5C06, 0) + table.put(0x7AEE, 0) + table.put(0x5DBA, 0) + table.put(0x5F62, 0) + table.put(0x6756, 0) + table.put(0x7C92, 0) + table.put(0x6753, 0) + table.put(0x3D96, 0) + table.put(0x393A, 0) + table.put(0x8D77, 0) + table.put(0x4227, 0) + table.put(0x5DB2, 0) + table.put(0x79EB, 0) + table.put(0x4CCE, 0) + table.put(0x674E, 0) + table.put(0x349E, 0) + table.put(0x5668, 0) + table.put(0x5306, 0) + table.put(0x7039, 0) + table.put(0x3A2E, 0) + table.put(0x5305, 0) + table.put(0x54BD, 0) + table.put(0x43D9, 0) + table.put(0x8910, 0) + table.put(0x43D5, 0) + table.put(0x5F53, 0) + table.put(0x285D2, 0) + table.put(0x6B9F, 0) + table.put(0x738B, 0) + table.put(0x26B3C, 0) + table.put(0x7C7B, 0) + table.put(0x26247, 0) + table.put(0x0313, 0) + table.put(0x7387, 0) + table.put(0x5651, 0) + table.put(0x23D1E, 0) + table.put(0x456B, 0) + table.put(0x7ACB, 0) + table.put(0x23ED1, 0) + table.put(0x82BD, 0) + table.put(0x391C, 0) + table.put(0x8103, 0) + table.put(0x24FB8, 0) + table.put(0x701E, 0) + table.put(0x54A2, 0) + table.put(0x701B, 0) + table.put(0x737A, 0) + table.put(0x4202, 0) + table.put(0x71D0, 0) + table.put(0x79CA, 0) + table.put(0x4561, 0) + table.put(0x82B3, 0) + table.put(0x0301, 0) + table.put(0x82B1, 0) + table.put(0x0300, 0) + table.put(0x71CE, 0) + table.put(0x7375, 0) + table.put(0x455D, 0) + table.put(0x96F7, 0) + table.put(0x96F6, 0) + table.put(0x8457, 0) + table.put(0x22331, 0) + table.put(0x60E1, 0) + table.put(0x7C60, 0) + table.put(0x6721, 0) + table.put(0x57F4, 0) + table.put(0x26CD5, 0) + table.put(0x24FA1, 0) + table.put(0x6B79, 0) + table.put(0x6B77, 0) + table.put(0x7AB1, 0) + table.put(0x26228, 0) + table.put(0x671B, 0) + table.put(0x4E41, 0) + table.put(0x6B72, 0) + table.put(0x60D8, 0) + table.put(0x9094, 0) + table.put(0x5F22, 0) + table.put(0x6717, 0) + table.put(0x27667, 0) + table.put(0x8449, 0) + table.put(0x96E3, 0) + table.put(0x4E3D, 0) + table.put(0x96E2, 0) + table.put(0x5197, 0) + table.put(0x9E1E, 0) + table.put(0x5195, 0) + table.put(0x5192, 0) + table.put(0x85FA, 0) + table.put(0x79AE, 0) + table.put(0x908F, 0) + table.put(0x4E39, 0) + table.put(0x25133, 0) + table.put(0x4E38, 0) + table.put(0x4E32, 0) + table.put(0x843D, 0) + table.put(0x518D, 0) + table.put(0x60C7, 0) + table.put(0x291DF, 0) + table.put(0x6700, 0) + table.put(0x5180, 0) + table.put(0x24C92, 0) + table.put(0x7C3E, 0) + table.put(0x907C, 0) + table.put(0x4E26, 0) + table.put(0x7DF4, 0) + table.put(0x7F9A, 0) + table.put(0x6B54, 0) + table.put(0x21B18, 0) + table.put(0x57CE, 0) + table.put(0x2003, 0) + table.put(0x2002, 0) + table.put(0x4096, 0) + table.put(0x5973, 0) + table.put(0x8ED4, 0) + table.put(0x2872E, 0) + table.put(0x8D1B, 0) + table.put(0x5F04, 0) + table.put(0x452B, 0) + table.put(0x625D, 0) + table.put(0x7F95, 0) + table.put(0x9072, 0) + table.put(0x20A2C, 0) + table.put(0x96C3, 0) + table.put(0x5177, 0) + table.put(0x8786, 0) + table.put(0x25119, 0) + table.put(0x3D33, 0) + table.put(0x5A66, 0) + table.put(0x2A600, 0) + table.put(0x5A62, 0) + table.put(0x8ECA, 0) + table.put(0x68A8, 0) + table.put(0x27F2F, 0) + table.put(0x516D, 0) + table.put(0x5606, 0) + table.put(0x5962, 0) + table.put(0x96B8, 0) + table.put(0x985E, 0) + table.put(0x7F85, 0) + table.put(0x5169, 0) + table.put(0x8779, 0) + table.put(0x4E0D, 0) + table.put(0x5168, 0) + table.put(0x5167, 0) + table.put(0x85CD, 0) + table.put(0x5164, 0) + table.put(0x8D08, 0) + table.put(0x6E9C, 0) + table.put(0x20122, 0) + table.put(0x732A, 0) + table.put(0x4FBF, 0) + table.put(0x7F7A, 0) + table.put(0x25BAB, 0) + table.put(0x4FBB, 0) + table.put(0x24814, 0) + table.put(0x876B, 0) + table.put(0x5954, 0) + table.put(0x46C7, 0) + table.put(0x5951, 0) + table.put(0x7F79, 0) + table.put(0x96A3, 0) + table.put(0x7F72, 0) + table.put(0x6CE5, 0) + table.put(0x5154, 0) + table.put(0x3781, 0) + table.put(0x5BFF, 0) + table.put(0x46BE, 0) + table.put(0x6599, 0) + table.put(0x7DC7, 0) + table.put(0x99FE, 0) + table.put(0x6234, 0) + table.put(0x4FAE, 0) + table.put(0x5948, 0) + table.put(0x514D, 0) + table.put(0x5944, 0) + table.put(0x6B21, 0) + table.put(0x2219F, 0) + table.put(0x5BF3, 0) + table.put(0x74CA, 0) + table.put(0x7DBE, 0) + table.put(0x622E, 0) + table.put(0x63E4, 0) + table.put(0x983B, 0) + table.put(0x99F1, 0) + table.put(0x5145, 0) + table.put(0x52FA, 0) + table.put(0x5BEE, 0) + table.put(0x5140, 0) + table.put(0x3EB8, 0) + table.put(0x549E, 0) + table.put(0x52F5, 0) + table.put(0x6CCD, 0) + table.put(0x7669, 0) + table.put(0x6CCC, 0) + table.put(0x5BE7, 0) + table.put(0x26523, 0) + table.put(0x9038, 0) + table.put(0x27966, 0) + table.put(0x621B, 0) + table.put(0x6E6E, 0) + table.put(0x5A27, 0) + table.put(0x3EAC, 0) + table.put(0x88FA, 0) + table.put(0x9829, 0) + table.put(0x6578, 0) + table.put(0x22183, 0) + table.put(0x88F8, 0) + table.put(0x6CBF, 0) + table.put(0x7DA0, 0) + table.put(0x6210, 0) + table.put(0x6B04, 0) + table.put(0x52E4, 0) + table.put(0x5BD8, 0) + table.put(0x216EA, 0) + table.put(0x4046, 0) + table.put(0x5922, 0) + table.put(0x5D7C, 0) + table.put(0x5A1B, 0) + table.put(0x9023, 0) + table.put(0x63C5, 0) + table.put(0x63C4, 0) + table.put(0x656C, 0) + table.put(0x829D, 0) + table.put(0x52DE, 0) + table.put(0x24C36, 0) + table.put(0x591A, 0) + table.put(0x9818, 0) + table.put(0x8B8A, 0) + table.put(0x7F3E, 0) + table.put(0x26FB1, 0) + table.put(0x91D1, 0) + table.put(0x6200, 0) + table.put(0x4039, 0) + table.put(0x238A7, 0) + table.put(0x8291, 0) + table.put(0x52D2, 0) + table.put(0x88E1, 0) + table.put(0x5D6E, 0) + table.put(0x91CF, 0) + table.put(0x5BC3, 0) + table.put(0x2541D, 0) + table.put(0x49E6, 0) + table.put(0x8B80, 0) + table.put(0x7642, 0) + table.put(0x5D6B, 0) + table.put(0x91CC, 0) + table.put(0x980B, 0) + table.put(0x8728, 0) + table.put(0x99C2, 0) + table.put(0x88DE, 0) + table.put(0x828B, 0) + table.put(0x3B9D, 0) + table.put(0x6556, 0) + table.put(0x52C9, 0) + table.put(0x52C7, 0) + table.put(0x88D7, 0) + table.put(0x2A105, 0) + table.put(0x6FFE, 0) + table.put(0x5906, 0) + table.put(0x719C, 0) + table.put(0x63A9, 0) + table.put(0x654F, 0) + table.put(0x9C57, 0) + table.put(0x5468, 0) + table.put(0x88CF, 0) + table.put(0x63A0, 0) + table.put(0x798F, 0) + table.put(0x798E, 0) + table.put(0x798D, 0) + table.put(0x91B4, 0) + table.put(0x8279, 0) + table.put(0x5D50, 0) + table.put(0x7A81, 0) + table.put(0x870E, 0) + table.put(0x38E3, 0) + table.put(0x4018, 0) + table.put(0x52B3, 0) + table.put(0x6FEB, 0) + table.put(0x88C2, 0) + table.put(0x66F8, 0) + table.put(0x99A7, 0) + table.put(0x66F4, 0) + table.put(0x826F, 0) + table.put(0x6E2F, 0) + table.put(0x9DFA, 0) + table.put(0x22BF1, 0) + table.put(0x9C40, 0) + table.put(0x761F, 0) + table.put(0x797F, 0) + table.put(0x761D, 0) + table.put(0x5EFE, 0) + table.put(0x5D43, 0) + table.put(0x934A, 0) + table.put(0x4AB2, 0) + table.put(0x9F9C, 0) + table.put(0x8B58, 0) + table.put(0x52A3, 0) + table.put(0x4008, 0) + table.put(0x6094, 0) + table.put(0x7610, 0) + table.put(0x688E, 0) + table.put(0x216A8, 0) + table.put(0x5448, 0) + table.put(0x80B2, 0) + table.put(0x233D5, 0) + table.put(0x6E1A, 0) + table.put(0x5442, 0) + table.put(0x5EEC, 0) + table.put(0x02B9, 0) + table.put(0x6885, 0) + table.put(0x38C7, 0) + table.put(0x9F8E, 0) + table.put(0x80AD, 0) + table.put(0x9F8D, 0) + table.put(0x6881, 0) + table.put(0x4359, 0) + table.put(0x716E, 0) + table.put(0x7965, 0) + table.put(0x23CBC, 0) + table.put(0x5438, 0) + table.put(0x6081, 0) + table.put(0x6FC6, 0) + table.put(0x261DA, 0) + table.put(0x233C3, 0) + table.put(0x795E, 0) + table.put(0x795D, 0) + table.put(0x250F3, 0) + table.put(0x250F2, 0) + table.put(0x8B39, 0) + table.put(0x5ED9, 0) + table.put(0x578B, 0) + table.put(0x55E2, 0) + table.put(0x7956, 0) + table.put(0x7A4F, 0) + table.put(0x9686, 0) + table.put(0x5ED3, 0) + table.put(0x5ED2, 0) + table.put(0x66C6, 0) + table.put(0x7950, 0) + table.put(0x7A4A, 0) + table.put(0x5D19, 0) + table.put(0x83F1, 0) + table.put(0x77A7, 0) + table.put(0x5ECA, 0) + table.put(0x967C, 0) + table.put(0x83EF, 0) + table.put(0x3B49, 0) + table.put(0x7A40, 0) + table.put(0x4F8B, 0) + table.put(0x541D, 0) + table.put(0x7949, 0) + table.put(0x5EC9, 0) + table.put(0x7948, 0) + table.put(0x9678, 0) + table.put(0x4334, 0) + table.put(0x9675, 0) + table.put(0x4F86, 0) + table.put(0x66B4, 0) + table.put(0x83E7, 0) + table.put(0x7149, 0) + table.put(0x4F80, 0) + table.put(0x7145, 0) + table.put(0x8CD3, 0) + table.put(0x793E, 0) + table.put(0x72FC, 0) + table.put(0x793C, 0) + table.put(0x9DB4, 0) + table.put(0x540F, 0) + table.put(0x6852, 0) + table.put(0x83DC, 0) + table.put(0x55C2, 0) + table.put(0x5EB6, 0) + table.put(0x55C0, 0) + table.put(0x5EB3, 0) + table.put(0x9304, 0) + table.put(0x5EB0, 0) + table.put(0x219C8, 0) + table.put(0x5406, 0) + table.put(0x8CC8, 0) + table.put(0x7498, 0) + table.put(0x208DE, 0) + table.put(0x8CC2, 0) + table.put(0x8CC1, 0) + table.put(0x792A, 0) + table.put(0x83CC, 0) + table.put(0x83CA, 0) + table.put(0x55B3, 0) + table.put(0x5EA6, 0) + table.put(0x8B01, 0) + table.put(0x83C9, 0) + table.put(0x7A1C, 0) + table.put(0x9F43, 0) + table.put(0x7489, 0) + table.put(0x4F60, 0) + table.put(0x27ED3, 0) + table.put(0x55AB, 0) + table.put(0x7485, 0) + table.put(0x26C36, 0) + table.put(0x61F6, 0) + table.put(0x964D, 0) + table.put(0x964B, 0) + table.put(0x61F2, 0) + table.put(0x3B19, 0) + table.put(0x8204, 0) + table.put(0x83BD, 0) + table.put(0x69EA, 0) + table.put(0x2633E, 0) + table.put(0x8564, 0) + table.put(0x9F3B, 0) + table.put(0x8201, 0) + table.put(0x7BC9, 0) + table.put(0x8CAB, 0) + table.put(0x97FF, 0) + table.put(0x7BC6, 0) + table.put(0x24608, 0) + table.put(0x4301, 0) + table.put(0x20B63, 0) + table.put(0x6C88, 0) + table.put(0x7BC0, 0) + table.put(0x6ADB, 0) + table.put(0x7471, 0) + table.put(0x7D63, 0) + table.put(0x3B08, 0) + table.put(0x29145, 0) + table.put(0x83AD, 0) + table.put(0x58F7, 0) + table.put(0x6AD3, 0) + table.put(0x25249, 0) + table.put(0x58F2, 0) + table.put(0x72C0, 0) + table.put(0x61DE, 0) + table.put(0x681F, 0) + table.put(0x7D5B, 0) + table.put(0x6383, 0) + table.put(0x7469, 0) + table.put(0x58EE, 0) + table.put(0x529B, 0) + table.put(0x2A291, 0) + table.put(0x9199, 0) + table.put(0x264DA, 0) + table.put(0x962E, 0) + table.put(0x6817, 0) + table.put(0x601C, 0) + table.put(0x637B, 0) + table.put(0x29496, 0) + table.put(0x97E0, 0) + table.put(0x21DE6, 0) + table.put(0x21DE4, 0) + table.put(0x5B85, 0) + table.put(0x745C, 0) + table.put(0x97DB, 0) + table.put(0x6012, 0) + table.put(0x50E7, 0) + table.put(0x6C67, 0) + table.put(0x9F16, 0) + table.put(0x58DF, 0) + table.put(0x72AF, 0) + table.put(0x5289, 0) + table.put(0x9F0F, 0) + table.put(0x58D8, 0) + table.put(0x214E4, 0) + table.put(0x25CD0, 0) + table.put(0x50DA, 0) + table.put(0x3FFC, 0) + table.put(0x6368, 0) + table.put(0x4635, 0) + table.put(0x4995, 0) + table.put(0x2A0CE, 0) + table.put(0x9F05, 0) + table.put(0x5717, 0) + table.put(0x9A6A, 0) + table.put(0x5716, 0) + table.put(0x7447, 0) + table.put(0x86E2, 0) + table.put(0x236A3, 0) + table.put(0x23AFA, 0) + table.put(0x6AA8, 0) + table.put(0x61B2, 0) + table.put(0x5277, 0) + table.put(0x50CF, 0) + table.put(0x36FC, 0) + table.put(0x6C4E, 0) + table.put(0x808B, 0) + table.put(0x5272, 0) + table.put(0x7D2F, 0) + table.put(0x61AF, 0) + table.put(0x916A, 0) + table.put(0x69A3, 0) + table.put(0x6350, 0) + table.put(0x4A76, 0) + table.put(0x61A4, 0) + table.put(0x6DF9, 0) + table.put(0x807E, 0) + table.put(0x7D22, 0) + table.put(0x36EE, 0) + table.put(0x8516, 0) + table.put(0x58B3, 0) + table.put(0x2339C, 0) + table.put(0x75E2, 0) + table.put(0x778B, 0) + table.put(0x4A6E, 0) + table.put(0x8070, 0) + table.put(0x8FB6, 0) + table.put(0x6DEA, 0) + table.put(0x58AC, 0) + table.put(0x23393, 0) + table.put(0x7422, 0) + table.put(0x8FB0, 0) + table.put(0x633D, 0) + table.put(0x806F, 0) + table.put(0x669C, 0) + table.put(0x7D10, 0) + table.put(0x58A8, 0) + table.put(0x8863, 0) + table.put(0x8860, 0) + table.put(0x26F2C, 0) + table.put(0x00C5, 0) + table.put(0x6691, 0) + table.put(0x387C, 0) + table.put(0x8060, 0) + table.put(0x6DDA, 0) + table.put(0x5B3E, 0) + table.put(0x3515, 0) + table.put(0x270D2, 0) + table.put(0x5246, 0) + table.put(0x7D00, 0) + return@lazy table + } +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/Exceptions.kt b/runtime/src/main/kotlin/kotlin/text/regex/Exceptions.kt new file mode 100644 index 00000000000..de2ba9bf401 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/Exceptions.kt @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Encapsulates a syntax error that occurred during the compilation of a + * [Pattern]. Might include a detailed description, the original regular + * expression, and the index at which the error occurred. + * + * @author Nikolay A. Kuznetsov + */ + +// TODO: Rewrite +class PatternSyntaxException(val description: String = "", val pattern: String = "", index: Int = -1) + : IllegalArgumentException("Error in \"$pattern\" ($index). $description") diff --git a/runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt b/runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt new file mode 100644 index 00000000000..a52b9fbbf78 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/Lexer.kt @@ -0,0 +1,1069 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// TODO: Licenses. + +package kotlin.text + +/** + * The purpose of this class is to break given pattern into RE tokens; + * @author Nikolay A. Kuznetsov + */ +// TODO: Move in C++. +typealias IntArrHash = HashMap +typealias IntHash = HashMap + +/** + * This is base class for special tokens like character classes and quantifiers. + * + * @author Nikolay A. Kuznetsov + */ +internal abstract class SpecialToken { + + /** + * Returns the type of the token, may return following values: + * TOK_CHARCLASS - token representing character class; + * TOK_QUANTIFIER - token representing quantifier; + */ + abstract val type: Type + + // TODO: Replace with an enum? + enum class Type { + CHARCLASS, + QUANTIFIER + } +} + +internal class Lexer(val patternString: String, flags: Int) { + + // The property is set in the init block after some transformations over the pattern string. + private val pattern: CharArray + + var flags = flags + private set + + // Modes =========================================================================================================== + enum class Mode { + PATTERN, + RANGE, + ESCAPE + } + + /** + * Mode: whether the lexer processes character range ([Mode.RANGE]), escaped sequence ([Mode.RANGE]) + * or any other part of a regex ([PATTERN]). + */ + var mode = Mode.PATTERN + private set + + /** When in [Mode.ESCAPE] mode, this field will save the previous one */ + private var savedMode = Mode.PATTERN + + // TODO: Remove it. + // TODO: Harmony doesn't allow us to set the ESCAPE mode from outside of the lexer. DO we need this limitation? + fun setModeWithReread(value: Mode) { + if(value == Mode.PATTERN || value == Mode.RANGE) { + mode = value + } + if (mode == Mode.PATTERN) { + reread() + } + } + + // Tokens ========================================================================================================== + // TODO: May be add getters. + internal var lookBack: Int = 0 // Previous char read. + private set + internal var currentChar: Int = 0 // Current character read. Returns 0 if there is no more characters. + private set + internal var lookAhead: Int = 0 // Next character. + private set + + // TODO: Investigate if we can make it not nullable or not + internal var curSpecialToken: SpecialToken? = null // Current special token (e.g. quantifier) + private set + internal var lookAheadSpecialToken: SpecialToken? = null // Next special token + private set + + // Indices in the pattern. + private var index = 0 // Current char being processed index. + private var prevNonWhitespaceIndex = 0 // Previous non-whitespace character index. + private var curTokenIndex = 0 // Current token start index. TODO: == index? + private var lookAheadTokenIndex = 0 // Next token index. + + init { + var processedPattern = patternString + if (flags and Pattern.LITERAL > 0) { + processedPattern = Pattern.quote(patternString) + } else if (flags and Pattern.CANON_EQ > 0) { + processedPattern = Lexer.normalize(patternString) // TODO: rewrite normalization + } + + this.pattern = processedPattern.toCharArray().copyOf(processedPattern.length + 2) + this.pattern[this.pattern.size - 1] = 0.toChar() + this.pattern[this.pattern.size - 2] = 0.toChar() + + // Read first two tokens. + movePointer() + movePointer() + } + + // Character checks ================================================================================================ + /** Returns true, if current token is special, i.e. quantifier, or other compound token. */ + val isSpecial: Boolean get() = curSpecialToken != null + val isQuantifier: Boolean get() = isSpecial && curSpecialToken!!.type == SpecialToken.Type.QUANTIFIER + val isNextSpecial: Boolean get() = lookAheadSpecialToken != null + + // TODO: May be replace with some already existing. + private fun Int.isSurrogatePair() : Boolean { + val high = (this ushr 16).toChar() + val low = this.toChar() + return high.isHighSurrogate() && low.isLowSurrogate() + } + + private fun Char.isLineSeparator(): Boolean = + this == '\n' || this == '\r' || this == '\u0085' || this.toInt() or 1 == '\u2029'.toInt() + + // TODO: Replace with properties + /** Checks if there are any characters in the pattern. */ + fun isEmpty(): Boolean = + currentChar == 0 && lookAhead == 0 && index >= pattern.size && !isSpecial + + /** Return true if the current character is letter, false otherwise .*/ + fun isLetter(): Boolean = + !isEmpty() && !isSpecial && isLetter(currentChar) + + /** Check if the current char is high/low surrogate. */ + fun isHighSurrogate(): Boolean = currentChar in 0xDBFF..0xD800 + fun isLowSurrogate(): Boolean = currentChar in 0xDFFF..0xDC00 + fun isSurrogate(): Boolean = isHighSurrogate() || isLowSurrogate() + + /** + * Restores flags for Lexer + * @param flags + */ + // TODO: Refactor. + fun restoreFlags(flags: Int) { + this.flags = flags + lookAhead = currentChar + lookAheadSpecialToken = curSpecialToken + + // curTokenIndex is an index of closing bracket ')' + index = curTokenIndex + 1 + lookAheadTokenIndex = curTokenIndex + movePointer() + } + + override fun toString(): String { + return patternString + } + + // Processing index moving ========================================================================================= + /** Returns current character and moves string index to the next one. */ + operator fun next(): Int { + movePointer() + return lookBack + } + + /** Returns current special token and moves string index to the next one */ + fun nextSpecial(): SpecialToken? { + val res = curSpecialToken + movePointer() + return res + } + + /** + * Reread current character. May be required if a previous token changes mode + * to one with different character interpretation. + */ + private fun reread() { + lookAhead = currentChar + lookAheadSpecialToken = curSpecialToken + index = lookAheadTokenIndex + lookAheadTokenIndex = curTokenIndex + movePointer() + } + + /** + * Returns the next character index to read and moves pointer to the next one. + * If comments flag is on this method will skip comments and whitespaces. + * + * The following actions are equivalent if comments flag is off: + * currentChar = pattern[index++] == currentChar = pattern[nextIndex] + */ + private fun nextIndex(): Int { + prevNonWhitespaceIndex = index + if (flags and Pattern.COMMENTS != 0) { // TODO: Refactor flags. + skipComments() + } else { + index++ + } + return prevNonWhitespaceIndex + } + + /** Skips comments and whitespaces */ + private fun skipComments(): Int { + val length = pattern.size - 2 + index++ + do { + while (index < length && pattern[index].isWhitespace()) { + index++ + } + if (index < length && pattern[index] == '#') { + index++ + while (index < length && !pattern[index].isLineSeparator()) { + index++ + } + } else { + return index + } + } while (true) + } + + /** + * Returns the next code point in the pattern string. + */ + private fun nextCodePoint(): Int { + val high = pattern[nextIndex()] // nextIndex skips comments and whitespaces if comments flag is on. + if (high.isHighSurrogate()) { + // Low and high chars may be delimited by spaces. + val lowExpectedIndex = prevNonWhitespaceIndex + 1 + if (lowExpectedIndex < pattern.size) { + val low = pattern[lowExpectedIndex] + if (low.isLowSurrogate()) { + nextIndex() + return Char.toCodePoint(high, low) + } + } + } + return high.toInt() + } + + /** + * Moves pointer one position right. Saves the current character to [lookBack], + * [lookAhead] to the current one and finally read one more to [lookAhead]. + */ + private fun movePointer() { + // swap pointers + lookBack = currentChar + currentChar = lookAhead + curSpecialToken = lookAheadSpecialToken + curTokenIndex = lookAheadTokenIndex + lookAheadTokenIndex = index + var reread: Boolean + do { + // Read the next character, analyze it and construct a token. + lookAhead = if (index < pattern.size) nextCodePoint() else 0 + lookAheadSpecialToken = null + + if (mode == Mode.ESCAPE) { + processInEscapeMode() + } + + reread = when (mode) { + Mode.PATTERN -> processInPatternMode() + Mode.RANGE -> processInRangeMode() + else -> false + } + } while (reread) + } + + // Special functions called from [movePointer] function to process chars in different modes ======================== + /** + * Processing an escaped sequence like "\Q foo \E". Just skip a character if it is not \E. + * Returns whether we need to reread the character or not + */ + private fun processInEscapeMode(): Boolean { + if (lookAhead == '\\'.toInt()) { + // Need not care about supplementary code points here. + val lookAheadChar: Char = if (index < pattern.size) pattern[nextIndex()] else '\u0000' + lookAhead = lookAheadChar.toInt() + + if (lookAheadChar == 'E') { + // If \E found - change the mode to the previous one and shift to the next char. + mode = savedMode + // TODO: + we create an array with 2 additional characters but check the index here. May be there is not need to check it? + lookAhead = if (index <= pattern.size - 2) nextCodePoint() else 0 + } else { + // If \ have no E - make a step back and return. + lookAhead = '\\'.toInt() + index = prevNonWhitespaceIndex + } + } + return false + } + + /** Processes a next character in [Mode.PATTERN] mode. Returns whether we need to reread the character or not */ + private fun processInPatternMode(): Boolean { + if (lookAhead.isSurrogatePair()) { + return false + } + val lookAheadChar = lookAhead.toChar() + + if (lookAheadChar == '\\') { + return processEscapedChar() + } + + when (lookAheadChar) { + // Quantifier (*, +, ?). TODO: May be create a quantifier here. + '+', '*', '?' -> { + val mode = if (index < pattern.size) pattern[index] else '*' + // look at the next character to determine if the mode is greedy, reluctant or possessive. + when (mode) { + '+' -> { lookAhead = lookAhead or Lexer.QMOD_POSSESSIVE; nextIndex() } + '?' -> { lookAhead = lookAhead or Lexer.QMOD_RELUCTANT; nextIndex() } + else -> lookAhead = lookAhead or Lexer.QMOD_GREEDY + } + } + + // Quantifier ({x,y}). + '{' -> lookAheadSpecialToken = processQuantifier() + + // $. + '$' -> lookAhead = CHAR_DOLLAR + + // A group or a special construction. + '(' -> { + if (pattern[index] != '?') { + // Group + lookAhead = CHAR_LEFT_PARENTHESIS + } else { + // Special constructs (non-capturing groups, look ahead/look behind etc). + nextIndex() + var char = pattern[index] + var isLookBehind = false + do { + if (!isLookBehind) { + when (char) { + // Look ahead or an atomic group. + '!' -> { lookAhead = CHAR_NEG_LOOKAHEAD; nextIndex() } + '=' -> { lookAhead = CHAR_POS_LOOKAHEAD; nextIndex() } + '>' -> { lookAhead = CHAR_ATOMIC_GROUP; nextIndex() } + // Positive / negaitve look behind - need to check the next char. + '<' -> { + nextIndex() + char = pattern[index] + isLookBehind = true + } + // Flags. + else -> { + lookAhead = readFlags() + + // We return `res = res or 1 shl 8` from readFlags() if we read (?idmsux-idmsux) + if (lookAhead >= 256) { + // Just flags (no non-capturing group with them). Erase auxiliary bit. + lookAhead = lookAhead and 0xff + flags = lookAhead + lookAhead = lookAhead shl 16 + lookAhead = CHAR_FLAGS or lookAhead + } else { + // A non-capturing group with flags: (?:Foo) + flags = lookAhead + lookAhead = lookAhead shl 16 + lookAhead = CHAR_NONCAP_GROUP or lookAhead + } + } + } + } else { + // Process the second char for look behind construction. + isLookBehind = false + when (char) { + '!' -> { lookAhead = CHAR_NEG_LOOKBEHIND; nextIndex() } + '=' -> { lookAhead = CHAR_POS_LOOKBEHIND; nextIndex() } + else -> throw PatternSyntaxException() + } + } + } while (isLookBehind) + } + } + + ')' -> lookAhead = CHAR_RIGHT_PARENTHESIS + '[' -> { lookAhead = CHAR_LEFT_SQUARE_BRACKET; mode = Mode.RANGE } + // TODO: No need in this case. + /*']' -> { + assert(mode == Mode.PATTERN) + /* // TODO: Think we cannot meet it here. because we already checked the mode and it is PATTERN, not RANGE. Check it. + if (mode == Lexer.MODE_RANGE) { + lookAhead = CHAR_RIGHT_SQUARE_BRACKET + }*/ + }*/ + '^' -> lookAhead = CHAR_CARET + '|' -> lookAhead = CHAR_VERTICAL_BAR + '.' -> lookAhead = CHAR_DOT + } + return false + } + + /** Processes a character inside a range. Returns whether we need to reread the character or not */ + private fun processInRangeMode(): Boolean { + if (lookAhead.isSurrogatePair()) { + return false + } + val lookAheadChar = lookAhead.toChar() + + when (lookAheadChar) { + '\\' -> return processEscapedChar() + '[' -> lookAhead = CHAR_LEFT_SQUARE_BRACKET + ']' -> lookAhead = CHAR_RIGHT_SQUARE_BRACKET + '^' -> lookAhead = CHAR_CARET + '&' -> lookAhead = CHAR_AMPERSAND + '-' -> lookAhead = CHAR_HYPHEN + } + return false + } + + /** Processes an escaped (\x) character in any mode. Returns whether we need to reread the character or not */ + private fun processEscapedChar() : Boolean { + lookAhead = if (index < pattern.size - 2) nextCodePoint() else throw PatternSyntaxException() + + // The current code point cannot be a surrogate pair because it is an escaped special one. + // Cast it to char or just skip it as if we pass through the else branch of the when below. + if (lookAhead.isSurrogatePair()) { + return false + } + val lookAheadChar = lookAhead.toChar() + + when (lookAheadChar) { + // Character class. + 'P', 'p' -> { + val cs = parseCharClassName() + val negative = lookAheadChar == 'P' + + // TODO: Harmony processes errors thrown from the getPredefinedClass + lookAheadSpecialToken = AbstractCharClass.getPredefinedClass(cs, negative) + lookAhead = 0 + } + + // Word/whitespace/digit. + 'w', 's', 'd', 'W', 'S', 'D' -> { + // TODO: Strange string creation. Can we do easily + // TODO: may be use parseCharClassName here? + lookAheadSpecialToken = AbstractCharClass.getPredefinedClass( + fromCharArray(pattern, prevNonWhitespaceIndex, 1), + false + ) + lookAhead = 0 + } + + // Enter in ESCAPE mode. Skip this \Q symbol. + 'Q' -> { + savedMode = mode + mode = Mode.ESCAPE + return true + } + + // Special characters like tab, new line etc. + 't' -> lookAhead = '\t'.toInt() + 'n' -> lookAhead = '\n'.toInt() + 'r' -> lookAhead = '\r'.toInt() + 'f' -> lookAhead = '\u000C'.toInt() + 'a' -> lookAhead = '\u0007'.toInt() + 'e' -> lookAhead = '\u001B'.toInt() + + // Back references to capturing groups. + '1', '2', '3', '4', '5', '6', '7', '8', '9' -> { + if (mode == Mode.PATTERN) { + lookAhead = 0x80000000.toInt() or lookAhead // Captured group reference is 0x80... + } + } + + // A literal: octal, hex, or hex unicode. + // TODO: May be use standard Kotlin methods here. + '0' -> lookAhead = readOctals() + 'x' -> lookAhead = readHex(2) + 'u' -> lookAhead = readHex(4) + + // Special characters like EOL, EOI etc + 'b' -> lookAhead = CHAR_WORD_BOUND + 'B' -> lookAhead = CHAR_NONWORD_BOUND + 'A' -> lookAhead = CHAR_START_OF_INPUT + 'G' -> lookAhead = CHAR_PREVIOUS_MATCH + 'Z' -> lookAhead = CHAR_END_OF_LINE + 'z' -> lookAhead = CHAR_END_OF_INPUT + + // \cx - A control character corresponding to x. + 'c' -> { + if (index < pattern.size - 2) { + //need not care about supplementary codepoints here + lookAhead = pattern[nextIndex()].toInt() and 0x1f + } else { + // TODO: Add messages to the exceptions. + throw PatternSyntaxException() + } + } + + // TODO: May be throw an exception in any not supported case? + 'C', 'E', 'F', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'R', 'T', 'U', 'V', 'X', 'Y', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'o', 'q', 'y' -> throw PatternSyntaxException() + } + return false + } + + /** Process [lookAhead] in assumption that it's quantifier. */ + private fun processQuantifier(): Quantifier { + assert(lookAhead == '{'.toInt()) + val sb = StringBuilder(4) + var min = -1 + var max = -1 + + // Obtain a min value. + //var lookAheadChar: Char = lookAhead.toChar() + // TODO: Check {} case + var char: Char = if (index < pattern.size) pattern[nextIndex()] else throw PatternSyntaxException() + while (char != '}') { + + if (char == ',' && min < 0) { + try { + val minParsed = sb.toString().toInt() + min = if (minParsed >= 0) minParsed else throw PatternSyntaxException() + sb.length = 0 + } catch (nfe: NumberFormatException) { + throw PatternSyntaxException() + } + } else { + sb.append(char) + } + char = if (index < pattern.size) pattern[nextIndex()] else break + } + + if (char != '}') { + throw PatternSyntaxException() + } + + // Obtain a max value, if it exists + if (sb.isNotEmpty()) { + try { + val maxParsed = sb.toString().toInt() + max = if (maxParsed >= 0) maxParsed else throw PatternSyntaxException() + if (min < 0) { + min = max + } + } catch (nfe: NumberFormatException) { + throw PatternSyntaxException() + } + } + + // TODO: Double check this contition. + if (min < 0 || max >=0 && max - min < 0) { + throw PatternSyntaxException() + } + + val mod = if (index < pattern.size) pattern[index] else '*' + when (mod) { + '+' -> { lookAhead = Lexer.QUANT_COMP_P; nextIndex() } + '?' -> { lookAhead = Lexer.QUANT_COMP_R; nextIndex() } + else -> lookAhead = Lexer.QUANT_COMP + } + return Quantifier(min, max) // TODO: Save mode (P, R, G) in the Quantifier + } + + // Reading methods for specific tokens ============================================================================= + /** Process expression flags given with (?idmsux-idmsux). Returns the flags processed. */ + private fun readFlags(): Int { + var positive = true + var result = flags + + while (index < pattern.size) { + val char = pattern[index] + when (char) { + '-' -> { + if (!positive) { + throw PatternSyntaxException() + } + positive = false + } + + 'i' -> result = if (positive) + result or Pattern.CASE_INSENSITIVE + else + result xor Pattern.CASE_INSENSITIVE and result + + 'd' -> result = if (positive) + result or Pattern.UNIX_LINES + else + result xor Pattern.UNIX_LINES and result + + 'm' -> result = if (positive) + result or Pattern.MULTILINE + else + result xor Pattern.MULTILINE and result + + 's' -> result = if (positive) + result or Pattern.DOTALL + else + result xor Pattern.DOTALL and result + + // We don't support UNICODE_CASE. + /*'u' -> result = if (positive) + result or Pattern.UNICODE_CASE + else + result xor Pattern.UNICODE_CASE and result*/ + + 'x' -> result = if (positive) + result or Pattern.COMMENTS + else + result xor Pattern.COMMENTS and result + + ':' -> { + nextIndex() + return result + } + + ')' -> { + nextIndex() + return result or (1 shl 8) + } + } + nextIndex() + } + throw PatternSyntaxException() + } + + /** Parse character classes names and verifies correction of the syntax */ + private fun parseCharClassName(): String { + val sb = StringBuilder(10) + if (index < pattern.size - 2) { + // one symbol family + if (pattern[index] != '{') { + // TODO: Replace such string formation with something like patter.substring(..) + return "Is${pattern[nextIndex()]}" + } + + nextIndex() // Skip '{' + var char = pattern[nextIndex()] + while (index < pattern.size - 2 && char != '}') { + sb.append(char) + char = pattern[nextIndex()] + } + if (char != '}') throw PatternSyntaxException() + } + if (sb.isEmpty()) throw PatternSyntaxException() + + val res = sb.toString() + return when { + res.length == 1 -> "Is$res" + res.length > 3 && (res.startsWith("Is") || res.startsWith("In")) -> res.substring(2) + else -> res + } + } + + /** Process hexadecimal integer. */ + private fun readHex(max: Int): Int { + val builder = StringBuilder(max) + val length = pattern.size - 2 + var i = 0 + while (i < max && index < length) { + builder.append(pattern[nextIndex()]) + i++ + } + if (i == max) { + try { + return builder.toString().toInt(16) + } catch (e: NumberFormatException) {} + } + throw PatternSyntaxException() + } + + /** Process octal integer. */ + // TODO: Replace with String.toInt(8) + private fun readOctals(): Int { + val length = pattern.size - 2 // TODO: Add a special method for check 'index < pattern.size - 2' + var result = 0 + // TODO: Add digit conversion. + var digit = digitOf(pattern[index], 8) + if (digit == -1) { + throw PatternSyntaxException() + } + val max = if (digit > 3) 2 else 3 + var i = 0 + while (i < max && index < length && digit != -1) { + result *= 8 + result += digit + nextIndex() + digit = digitOf(pattern[index], 8) + i++ + } + return result + } + + companion object { + // Special characters. + val CHAR_DOLLAR = 0xe0000000.toInt() or '$'.toInt() + val CHAR_RIGHT_PARENTHESIS = 0xe0000000.toInt() or ')'.toInt() + val CHAR_LEFT_SQUARE_BRACKET = 0xe0000000.toInt() or '['.toInt() + val CHAR_RIGHT_SQUARE_BRACKET = 0xe0000000.toInt() or ']'.toInt() + val CHAR_CARET = 0xe0000000.toInt() or '^'.toInt() + val CHAR_VERTICAL_BAR = 0xe0000000.toInt() or '|'.toInt() + val CHAR_AMPERSAND = 0xe0000000.toInt() or '&'.toInt() + val CHAR_HYPHEN = 0xe0000000.toInt() or '-'.toInt() + val CHAR_DOT = 0xe0000000.toInt() or '.'.toInt() + val CHAR_LEFT_PARENTHESIS = 0x80000000.toInt() or '('.toInt() + val CHAR_NONCAP_GROUP = 0xc0000000.toInt() or '('.toInt() + val CHAR_POS_LOOKAHEAD = 0xe0000000.toInt() or '('.toInt() + val CHAR_NEG_LOOKAHEAD = 0xf0000000.toInt() or '('.toInt() + val CHAR_POS_LOOKBEHIND = 0xf8000000.toInt() or '('.toInt() + val CHAR_NEG_LOOKBEHIND = 0xfc000000.toInt() or '('.toInt() + val CHAR_ATOMIC_GROUP = 0xfe000000.toInt() or '('.toInt() + val CHAR_FLAGS = 0xff000000.toInt() or '('.toInt() + val CHAR_START_OF_INPUT = 0x80000000.toInt() or 'A'.toInt() + val CHAR_WORD_BOUND = 0x80000000.toInt() or 'b'.toInt() + val CHAR_NONWORD_BOUND = 0x80000000.toInt() or 'B'.toInt() + val CHAR_PREVIOUS_MATCH = 0x80000000.toInt() or 'G'.toInt() + val CHAR_END_OF_INPUT = 0x80000000.toInt() or 'z'.toInt() + val CHAR_END_OF_LINE = 0x80000000.toInt() or 'Z'.toInt() + + // Quantifier modes. + val QMOD_GREEDY = 0xe0000000.toInt() + val QMOD_RELUCTANT = 0xc0000000.toInt() + val QMOD_POSSESSIVE = 0x80000000.toInt() + + // Quantifiers. + val QUANT_STAR = QMOD_GREEDY or '*'.toInt() + val QUANT_STAR_P = QMOD_POSSESSIVE or '*'.toInt() + val QUANT_STAR_R = QMOD_RELUCTANT or '*'.toInt() + val QUANT_PLUS = QMOD_GREEDY or '+'.toInt() + val QUANT_PLUS_P = QMOD_POSSESSIVE or '+'.toInt() + val QUANT_PLUS_R = QMOD_RELUCTANT or '+'.toInt() + val QUANT_ALT = QMOD_GREEDY or '?'.toInt() + val QUANT_ALT_P = QMOD_POSSESSIVE or '?'.toInt() + val QUANT_ALT_R = QMOD_RELUCTANT or '?'.toInt() + val QUANT_COMP = QMOD_GREEDY or '{'.toInt() + val QUANT_COMP_P = QMOD_POSSESSIVE or '{'.toInt() + val QUANT_COMP_R = QMOD_RELUCTANT or '{'.toInt() + + /** Returns true if [ch] is a plain token. */ // TODO: May be rename + fun isLetter(ch: Int): Boolean { + // All supplementary codepoints have integer value that is >= 0. + return ch >= 0 + } + + // Decomposition =============================================================================================== + // TODO: Move to C++ side. + // Maximum length of decomposition. + val MAX_DECOMPOSITION_LENGTH = 4 + // Maximum length of Hangul decomposition. Note that MAX_HANGUL_DECOMPOSITION_LENGTH <= MAX_DECOMPOSITION_LENGTH. + val MAX_HANGUL_DECOMPOSITION_LENGTH = 3 + + /* + * Following constants are needed for Hangul canonical decomposition. + * Hangul decomposition algorithm and constants are taken according + * to description at http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf + * "3.12 Conjoining Jamo Behavior" + */ + val SBase = 0xAC00 + val LBase = 0x1100 + val VBase = 0x1161 + val TBase = 0x11A7 + val SCount = 11172 + val LCount = 19 + val VCount = 21 + val TCount = 28 + val NCount = 588 + + // TODO: Use decomposition tables on C++ side. + // TODO: Harmony uses its own IntHash and IntArrHash classes here. + // Canonical decomposition mappings + private var decompTable: IntArrHash? = null + + // TODO: Decompositions: + // Table that contains canonical combining classes + private var canonClassesTable: IntHash? = null + private var canonClassesTableSize: Int = 0 + + /* + * Table that contains information about Unicode codepoints with + * single codepoint decomposition + */ + private var singleDecompTable: IntHash? = null + private var singleDecompTableSize: Int = 0 + + private fun String.codePointAt(index: Int): Int { + val high = this[index] + if (high.isHighSurrogate() && index + 1 < this.length) { + val low = this[index + 1] + if (low.isLowSurrogate()) { + return Char.toCodePoint(high, low) + } + } + return high.toInt() + } + + /** + * Normalize given expression. + + * @param input - expression to normalize + * * + * @return normalized expression. + */ + // TODO: Implement on the C++ side. + fun normalize(input: String): String { + val inputChars = input.toCharArray() + val inputLength = inputChars.size + var resCodePointsIndex = 0 + var inputCodePointsIndex = 0 + var decompHangulIndex = 0 + + //codePoints of input + val inputCodePoints = IntArray(inputLength) + + //result of canonical decomposition of input + var resCodePoints = IntArray(inputLength * MAX_DECOMPOSITION_LENGTH) + + //current symbol's codepoint + var ch: Int + + //current symbol's decomposition + var decomp: IntArray? + + //result of canonical and Hangul decomposition of input + val decompHangul: IntArray + + //result of canonical decomposition of input in UTF-16 encoding + val result = StringBuilder() + + decompTable = HashDecompositions.hashDecompositions + canonClassesTable = CanClasses.hashCanClasses + canonClassesTableSize = canonClassesTable!!.size + singleDecompTable = SingleDecompositions.hashSingleDecompositions + singleDecompTableSize = singleDecompTable!!.size + + run { + var i = 0 + while (i < inputLength) { + ch = input.codePointAt(i) + inputCodePoints[inputCodePointsIndex++] = ch + i += if (Char.isSupplementaryCodePoint(ch)) 2 else 1 + } + } + + /* + * Canonical decomposition based on mappings in decompTable + */ + for (i in 0..inputCodePointsIndex - 1) { + ch = inputCodePoints[i] + + decomp = Lexer.getDecomposition(ch) + if (decomp == null) { + resCodePoints[resCodePointsIndex++] = ch + } else { + val curSymbDecompLength = decomp.size + + for (j in 0..curSymbDecompLength - 1) { + resCodePoints[resCodePointsIndex++] = decomp[j] + } + } + } + + /* + * Canonical ordering. + * See http://www.unicode.org/reports/tr15/#Decomposition for + * details + */ + resCodePoints = Lexer.getCanonicalOrder(resCodePoints, resCodePointsIndex) + + /* + * Decomposition for Hangul syllables. + * See http://www.unicode.org/reports/tr15/#Hangul for + * details + */ + decompHangul = IntArray(resCodePoints.size) + + for (i in 0..resCodePointsIndex - 1) { + val curSymb = resCodePoints[i] + + decomp = getHangulDecomposition(curSymb) + if (decomp == null) { + decompHangul[decompHangulIndex++] = curSymb + } else { + + /* + * Note that Hangul decompositions have length that is + * equal 2 or 3. + */ + decompHangul[decompHangulIndex++] = decomp[0] + decompHangul[decompHangulIndex++] = decomp[1] + if (decomp.size == 3) { + decompHangul[decompHangulIndex++] = decomp[2] + } + } + } + + /* + * Translating into UTF-16 encoding + */ + for (i in 0..decompHangulIndex - 1) { + result.append(Char.toChars(decompHangul[i])) + } + + return result.toString() + } + + /** + * Rearrange codepoints according + * to canonical order. + + * @param inputInts - array that contains Unicode codepoints + * * + * @param length - index of last Unicode codepoint plus 1 + * * + * * + * @return array that contains rearranged codepoints. + */ + // TODO: C++ side! + fun getCanonicalOrder(inputInts: IntArray, length: Int): IntArray { + val inputLength = if (length < inputInts.size) + length + else + inputInts.size + + /* + * Simple bubble-sort algorithm. + * Note that many codepoints have 0 + * canonical class, so this algorithm works + * almost lineary in overwhelming majority + * of cases. This is due to specific of Unicode + * combining classes and codepoints. + */ + for (i in 1..inputLength - 1) { + var j = i - 1 + val iCanonicalClass = getCanonicalClass(inputInts[i]) + val ch: Int + + if (iCanonicalClass == 0) { + continue + } + + while (j > -1) { + if (getCanonicalClass(inputInts[j]) > iCanonicalClass) { + j = j - 1 + } else { + break + } + } + + ch = inputInts[i] + for (k in i downTo j + 1 + 1) { + inputInts[k] = inputInts[k - 1] + } + inputInts[j + 1] = ch + } + + return inputInts + } + + + /** + * Gets decomposition for given codepoint from + * decomposition mappings table. + + * @param ch - Unicode codepoint + * * + * @return array of codepoints that is a canonical + * * decomposition of currentChar. + */ + // TODO @C++. + fun getDecomposition(ch: Int): IntArray? { + return decompTable!!.get(ch) + } + + /** + * Gets decomposition for given Hangul syllable. + * This is an implementation of Hangul decomposition algorithm + * according to http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf + * "3.12 Conjoining Jamo Behavior". + + * @param ch - given Hangul syllable + * * + * @return canonical decomposition of currentChar. + */ + // TODO: C++ + fun getHangulDecomposition(ch: Int): IntArray? { + val SIndex = ch - SBase + + if (SIndex < 0 || SIndex >= SCount) { + return null + } else { + val L = LBase + SIndex / NCount + val V = VBase + SIndex % NCount / TCount + var T = SIndex % TCount + val decomp: IntArray + + if (T == 0) { + decomp = intArrayOf(L, V) + } else { + T = TBase + T + decomp = intArrayOf(L, V, T) + } + return decomp + } + } + + /** + * Gets canonical class for given codepoint from + * decomposition mappings table. + + * @param - currentChar Unicode codepoint + * * + * @return canonical class for given Unicode codepoint + * * that is represented by currentChar. + */ + // TODO: C++ + fun getCanonicalClass(ch: Int): Int { + return canonClassesTable!!.get(ch) ?: 0 + } + + /** + * Tests if given codepoint is a canonical decomposition of another + * codepoint. + + * @param ch - codepoint to test + * * + * @return true if currentChar is a decomposition. + */ + // TODO: C++. Called from Pattern + fun hasSingleCodepointDecomposition(ch: Int): Boolean = singleDecompTable!!.get(ch) != null + + /** + * Tests if given codepoint has canonical decomposition + * and given codepoint's canonical class is not 0. + + * @param ch - codepoint to test + * * + * @return true if canonical class is not 0 and currentChar has a decomposition. + */ + // TODO: It's Ok to have this here. Called from DecomposedCharSet. + fun hasDecompositionNonNullCanClass(ch: Int): Boolean { + return (ch == 0x0340) or (ch == 0x0341) or (ch == 0x0343) or (ch == 0x0344) + } + + /** + * Tests Unicode codepoint if it is a boundary + * of decomposed Unicode codepoint. + + * @param ch - Unicode codepoint to test + * * + * @return true if given codepoint is a boundary. + */ + // TODO: Move to C++ side. + fun isDecomposedCharBoundary(ch: Int): Boolean { + val canClass = canonClassesTable!!.get(ch) + + //Lexer.getCanonicalClass(currentChar) == 0 + val isBoundary = canClass == null + + return isBoundary + } + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/MatchResult.kt b/runtime/src/main/kotlin/kotlin/text/regex/MatchResult.kt new file mode 100644 index 00000000000..7c4019dbc48 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/MatchResult.kt @@ -0,0 +1,125 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents a collection of captured groups in a single match of a regular expression. + * + * This collection has size of `groupCount + 1` where `groupCount` is the count of groups in the regular expression. + * Groups are indexed from 1 to `groupCount` and group with the index 0 corresponds to the entire match. + * + * An element of the collection at the particular index can be `null`, + * if the corresponding group in the regular expression is optional and + * there was no match captured by that group. + */ +// TODO: Add MatchGroup +interface MatchGroupCollection : Collection { + + /** Returns a group with the specified [index]. + * + * @return An instance of [MatchGroup] if the group with the specified [index] was matched or `null` otherwise. + * + * Groups are indexed from 1 to the count of groups in the regular expression. A group with the index 0 + * corresponds to the entire match. + */ + operator fun get(index: Int): MatchGroup? +} + +/** + * Extends [MatchGroupCollection] by introducing a way to get matched groups by name, when regex supports it. + */ +@SinceKotlin("1.1") interface MatchNamedGroupCollection : MatchGroupCollection { + /** + * Returns a named group with the specified [name]. + * @return An instance of [MatchGroup] if the group with the specified [name] was matched or `null` otherwise. + */ + operator fun get(name: String): MatchGroup? +} + +/** + * Represents the results from a single regular expression match. + */ +interface MatchResult { + /** The range of indices in the original string where match was captured. */ + val range: IntRange + /** The substring from the input string captured by this match. */ + val value: String + /** + * A collection of groups matched by the regular expression. + * + * This collection has size of `groupCount + 1` where `groupCount` is the count of groups in the regular expression. + * Groups are indexed from 1 to `groupCount` and group with the index 0 corresponds to the entire match. + */ + val groups: MatchGroupCollection + /** + * A list of matched indexed group values. + * + * This list has size of `groupCount + 1` where `groupCount` is the count of groups in the regular expression. + * Groups are indexed from 1 to `groupCount` and group with the index 0 corresponds to the entire match. + * + * If the group in the regular expression is optional and there were no match captured by that group, + * corresponding item in [groupValues] is an empty string. + * + * @sample: samples.text.Regexps.matchDestructuringToGroupValues + */ + val groupValues: List + + /** + * An instance of [MatchResult.Destructured] wrapper providing components for destructuring assignment of group values. + * + * component1 corresponds to the value of the first group, component2 — of the second, and so on. + * + * @sample: samples.text.Regexps.matchDestructuring + */ + val destructured: Destructured get() = Destructured(this) + + /** Returns a new [MatchResult] with the results for the next match, starting at the position + * at which the last match ended (at the character after the last matched character). + */ + fun next(): MatchResult? + + /** + * Provides components for destructuring assignment of group values. + * + * [component1] corresponds to the value of the first group, [component2] — of the second, and so on. + * + * If the group in the regular expression is optional and there were no match captured by that group, + * corresponding component value is an empty string. + * + * @sample: samples.text.Regexps.matchDestructuringToGroupValues + */ + class Destructured internal constructor(val match: MatchResult) { + + operator inline fun component1(): String = match.groupValues[1] + operator inline fun component2(): String = match.groupValues[2] + operator inline fun component3(): String = match.groupValues[3] + operator inline fun component4(): String = match.groupValues[4] + operator inline fun component5(): String = match.groupValues[5] + operator inline fun component6(): String = match.groupValues[6] + operator inline fun component7(): String = match.groupValues[7] + operator inline fun component8(): String = match.groupValues[8] + operator inline fun component9(): String = match.groupValues[9] + operator inline fun component10(): String = match.groupValues[10] + /** + * Returns destructured group values as a list of strings. + * First value in the returned list corresponds to the value of the first group, and so on. + * + * @sample: samples.text.Regexps.matchDestructuringToGroupValues + */ + fun toList(): List = match.groupValues.subList(1, match.groupValues.size) + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt b/runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt new file mode 100644 index 00000000000..cbc0a2e86f4 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/MatchResultImpl.kt @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Match result implementation + * @author Nikolay A. Kuznetsov + */ + +// TODO: rework defaults. +// TODO: May be we can remove left/rightBound and just create a substring for matching/searching. +// TODO: We don't use right/leftBound in our API now. So this function can be removed. +// TODO: But I think it may be useful to save them for further improvements of the API. +internal class MatchResultImpl +// TODO: Refactor the constructor comment. +/** + * @param input an input sequence for matching/searching. + * @param regex a [Regex] instance used for matching/searching. + * @param rightBound index in the [input] used as a right bound for matching/searching. Exclusive. + */ +constructor (internal val input: CharSequence, + internal val regex: Regex) : MatchResult { + + // Harmony's implementation ======================================================================================== + // TODO: Rework number of groups + private val nativePattern = regex.nativePattern + private val groupCount = nativePattern.capturingGroupCount + private val groupBounds = IntArray(groupCount * 2) { -1 } + + private val consumers = IntArray(nativePattern.consumersCount + 1) { -1 } // TODO: What is consumers array? + + // Used by quantifiers to store a count of a quantified expression occurrences. + val enterCounters: IntArray = IntArray( maxOf(nativePattern.groupQuantifierCount, 0) ) // TODO remove maxOf + + var startIndex: Int = 0 + set (startIndex: Int) { + field = startIndex + // TODO: what is it? And do we need it? + if (previousMatch < 0) { + previousMatch = startIndex + } + } + + var previousMatch = -1 + var mode = Regex.Mode.MATCH + + // MatchResult interface =========================================================================================== + /** The range of indices in the original string where match was captured. */ + override val range: IntRange + get() = getStart(0) until getEnd(0) // TODO: I don't like these get... set... methods. Rename them or make properties. + + /** The substring from the input string captured by this match. */ + override val value: String + get() = group(0) ?: throw AssertionError("No groupIndex #0 in the match result.") + + /** + * A collection of groups matched by the regular expression. + * + * This collection has size of `groupCount + 1` where `groupCount` is the count of groups in the regular expression. + * Groups are indexed from 1 to `groupCount` and group with the index 0 corresponds to the entire match. + */ + // Create one object or several ones? + override val groups: MatchGroupCollection = object: MatchGroupCollection, AbstractCollection() { + override val size: Int + get() = this@MatchResultImpl.groupCount + + + override fun iterator(): Iterator { + return object: Iterator { + var nextIndex: Int = 0 + + override fun hasNext(): Boolean { + return nextIndex < size + } + + override fun next(): MatchGroup? { + if (!hasNext()) { + throw NoSuchElementException() + } + return get(nextIndex++) + } + } + } + + override fun get(index: Int): MatchGroup? { + val value = group(index) ?: return null + return MatchGroup(value, getStart(index) until getEnd(index)) + } + } + + /** + * A list of matched indexed group values. + * + * This list has size of `groupCount + 1` where `groupCount` is the count of groups in the regular expression. + * Groups are indexed from 1 to `groupCount` and group with the index 0 corresponds to the entire match. + * + * If the group in the regular expression is optional and there were no match captured by that group, + * corresponding item in [groupValues] is an empty string. + * + * @sample: samples.text.Regexps.matchDestructuringToGroupValues + */ + override val groupValues: List + get() = mutableListOf().apply { + for (i in 0 until groupCount) { + this.add(group(i) ?: "") + } + } + + override fun next(): MatchResult? { + var nextStart = range.endInclusive + 1 + if (nextStart == input.length) { + return null + } + // If the current match is empty - shift by 1. + if (nextStart == range.start) { + nextStart++ + } + return regex.find(input, nextStart) + } + // ================================================================================================================= + + + // Harmony's implementation ======================================================================================== + // TODO: What is it? + fun setConsumed(counter: Int, value: Int) { + this.consumers[counter] = value + } + + fun getConsumed(counter: Int): Int { + return this.consumers[counter] + } + + fun isCaptured(group: Int): Boolean = getStart(group) >= 0 + + // Setters and getters for starts and ends of groups =============================================================== + // TODO: Decide what access modifier should we use in this class. + internal fun setStart(group: Int, offset: Int) { + checkGroup(group) + groupBounds[group * 2] = offset + } + + internal fun setEnd(group: Int, offset: Int) { + checkGroup(group) + groupBounds[group * 2 + 1] = offset + } + + /** + * Returns the index of the first character of the text that matched a given group. + * + * @param group the group, ranging from 0 to groupCount() - 1, with 0 representing the whole pattern. + * @return the character index. + */ + fun getStart(group: Int = 0): Int { + checkGroup(group) + return groupBounds[group * 2] + } + + /** + * Returns the index of the first character following the text that matched a given group. + * + * @param group the group, ranging from 0 to groupCount() - 1, with 0 representing the whole pattern. + * @return the character index. + */ + fun getEnd(group: Int = 0): Int { + checkGroup(group) + return groupBounds[group * 2 + 1] + } + + // ================================================================================================== + + /** + * Returns the text that matched a given group of the regular expression. + * + * @param group the group, ranging from 0 to groupCount() - 1, with 0 representing the whole pattern. + * @return the text that matched the group. + */ + // TODO: Reread and maybe rewrite. + fun group(group: Int = 0): String? { + val start = getStart(group) + val end = getEnd(group) + if (start < 0) { // TODO: Do we need to check end? + return null + } + assert(start <= end && end >= 0 && end <= input.length) // TODO: remove the assert. + return input.subSequence(getStart(group), getEnd(group)).toString() + } + + /** + * Returns the number of groups in the result, which is always equal to + * the number of groups in the original regular expression. + * + * @return the number of groups. + */ + fun groupCount(): Int { + return groupCount - 1 // TODO: Do something with it. These +/-1 are terrible. + } + + /* + * This method being called after any successful match; For now it's being + * used to check zero group for empty match; + */ + fun finalizeMatch() { + if (this.groupBounds[0] == -1) { + this.groupBounds[0] = this.startIndex + this.groupBounds[1] = this.startIndex + } + + previousMatch = getEnd() + } + + private fun checkGroup(group: Int) { + if (group < 0 || group > groupCount) { + throw IndexOutOfBoundsException() + } + } + + fun updateGroup(index: Int, srtOffset: Int, endOffset: Int) { + checkGroup(index) + groupBounds[index * 2] = srtOffset + groupBounds[index * 2 + 1] = endOffset + } +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/Pattern.kt b/runtime/src/main/kotlin/kotlin/text/regex/Pattern.kt new file mode 100644 index 00000000000..cfe2f157047 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/Pattern.kt @@ -0,0 +1,868 @@ +package kotlin.text + +/** + * Apache Harmony Pattern class implementation. + * Analogue for Kotlin Regex class. Need to be merged with it. + */ + +/** Represents a compiled pattern used by [Regex] for matching, searching, or replacing strings. */ +internal class Pattern(val pattern: String, flags: Int = 0) { + + var flags = flags + private set + + /** A lexer instance used to get tokens from the pattern. */ + private val lexemes = Lexer(pattern, flags) + + /* All back references that may be used in pattern. */ + private val backRefs = arrayOfNulls(BACK_REF_NUMBER) + + /** Is true if back referenced sets replacement by second compilation pass is needed.*/ + private var needsBackRefReplacement = false + + /** Global count of found capturing groups. */ + var capturingGroupCount = 0 + private set + + /** A number of group quantifiers in the pattern */ + var groupQuantifierCount = 0 + private set + + /** + * A number of consumers found in the pattern. + * Consumer is any expression ending with an FSet except capturing groups (they are counted by [capturingGroupCount]) + */ + var consumersCount = 0 + private set + + /** A node to start a matching/searching process by call startNode.matches/startNode.find. */ + internal val startNode: AbstractSet + + /** Compiles the given pattern */ + init { + if (flags != 0 && flags or flagsBitMask != flagsBitMask) { + throw IllegalArgumentException() + } + AbstractSet.counter = 1 // TODO: It's for debug purposes. + + startNode = processExpression(-1, this.flags, null) + + if (!lexemes.isEmpty()) { + throw PatternSyntaxException() + } + + // Finalize compilation + if (needsBackRefReplacement) { + startNode.processSecondPass() + } + } + + override fun toString(): String = pattern + + /** Return true if the pattern has the specified flag */ + private fun hasFlag(flag: Int): Boolean = flags and flag == flag + + // Compilation methods. ============================================================================================ + /** A->(a|)+ */ + private fun processAlternations(last: AbstractSet): AbstractSet { + val auxRange = CharClass(hasFlag(Pattern.CASE_INSENSITIVE)) + while (!lexemes.isEmpty() && lexemes.isLetter() + && (lexemes.lookAhead == 0 + || lexemes.lookAhead == Lexer.CHAR_VERTICAL_BAR + || lexemes.lookAhead == Lexer.CHAR_RIGHT_PARENTHESIS)) { + auxRange.add(lexemes.next()) + if (lexemes.currentChar == Lexer.CHAR_VERTICAL_BAR) { + lexemes.next() + } + } + val rangeSet = processRangeSet(auxRange) + rangeSet.next = last + + return rangeSet + } + + /** E->AE; E->S|E; E->S; A->(a|)+ E->S(|S)* */ + private fun processExpression(ch: Int, newFlags: Int, last: AbstractSet?): AbstractSet { + val children = ArrayList() + val savedFlags = flags + var saveChangedFlags = false + + if (newFlags != flags) { + flags = newFlags + } + + // Create a right finalizing set. + val fSet: FSet + when (ch) { + // Special groups: non-capturing, look ahead/behind etc. + Lexer.CHAR_NONCAP_GROUP -> fSet = NonCapFSet(consumersCount++) + Lexer.CHAR_POS_LOOKAHEAD, + Lexer.CHAR_NEG_LOOKAHEAD -> fSet = AheadFSet() + Lexer.CHAR_POS_LOOKBEHIND, + Lexer.CHAR_NEG_LOOKBEHIND -> fSet = BehindFSet(consumersCount++) + Lexer.CHAR_ATOMIC_GROUP -> fSet = AtomicFSet(consumersCount++) + // A Capturing group. + else -> { + if (last == null) { + // Whole pattern - group #0. + fSet = FinalSet() + saveChangedFlags = true + } else { + // TODO: Looks like next here is null. Or it is set later. + fSet = FSet(capturingGroupCount) + } + if (capturingGroupCount < BACK_REF_NUMBER) { + backRefs[capturingGroupCount] = fSet + } + capturingGroupCount++ + } + } + + //Process to EOF or ')' + do { + val child: AbstractSet + when { + // a|... + lexemes.isLetter() && lexemes.lookAhead == Lexer.CHAR_VERTICAL_BAR -> child = processAlternations(fSet) + // ..|.., e.g. in "a||||b" + lexemes.currentChar == Lexer.CHAR_VERTICAL_BAR -> { + child = EmptySet(fSet) + lexemes.next() + } + else -> { + child = processSubExpression(fSet) + if (lexemes.currentChar == Lexer.CHAR_VERTICAL_BAR) { + lexemes.next() + } + } + } + // TODO: Do we realy need this null check? Looks like child cannot be null. + children.add(child) + } while (!(lexemes.isEmpty() || lexemes.currentChar == Lexer.CHAR_RIGHT_PARENTHESIS)) + + // |) or | - add an empty node. + if (lexemes.lookBack == Lexer.CHAR_VERTICAL_BAR) { + children.add(EmptySet(fSet)) + } + + // Restore flags. + if (flags != savedFlags && !saveChangedFlags) { + flags = savedFlags + lexemes.restoreFlags(flags) + } + + when (ch) { + Lexer.CHAR_NONCAP_GROUP -> return NonCapturingJointSet(children, fSet) + Lexer.CHAR_POS_LOOKAHEAD -> return PositiveLookAheadSet(children, fSet) + Lexer.CHAR_NEG_LOOKAHEAD -> return NegativeLookAheadSet(children, fSet) + Lexer.CHAR_POS_LOOKBEHIND -> return PositiveLookBehindSet(children, fSet) + Lexer.CHAR_NEG_LOOKBEHIND -> return NegativeLookBehindSet(children, fSet) + Lexer.CHAR_ATOMIC_GROUP -> return AtomicJointSet(children, fSet) + + else -> when (children.size) { + 0 -> return EmptySet(fSet) + 1 -> return SingleSet(children[0], fSet) + else -> return JointSet(children, fSet) + } + } + } + + + /** + * T->aaa + */ + private fun processSequence(): AbstractSet { + val substring = StringBuilder() + while (!lexemes.isEmpty() + && lexemes.isLetter() + && !lexemes.isSurrogate() + && (!lexemes.isNextSpecial && lexemes.lookAhead == 0 // End of a pattern. + || !lexemes.isNextSpecial && Lexer.isLetter(lexemes.lookAhead) + || lexemes.lookAhead == Lexer.CHAR_RIGHT_PARENTHESIS + || lexemes.lookAhead and 0x8000ffff.toInt() == Lexer.CHAR_LEFT_PARENTHESIS + || lexemes.lookAhead == Lexer.CHAR_VERTICAL_BAR + || lexemes.lookAhead == Lexer.CHAR_DOLLAR)) { + val ch = lexemes.next() + + // TODO: Adapt to our checks. + if (Char.isSupplementaryCodePoint(ch)) { + substring.append(Char.toChars(ch)) + } else { + substring.append(ch.toChar()) + } + } + return SequenceSet(substring, hasFlag(CASE_INSENSITIVE)) + } + + /** + * D->a + */ + // TODO: Refactor + private fun processDecomposedChar(): AbstractSet { + val codePoints = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH) + val codePointsHangul: CharArray + var readCodePoints = 0 + var curSymb = -1 + var curSymbIndex = -1 + + if (!lexemes.isEmpty() && lexemes.isLetter()) { + curSymb = lexemes.next() + codePoints[readCodePoints] = curSymb + curSymbIndex = curSymb - Lexer.LBase + } + + /* + * We process decomposed Hangul syllable LV or LVT or process jamo L. + * See http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf + * "3.12 Conjoining Jamo Behavior" + */ + if (curSymbIndex >= 0 && curSymbIndex < Lexer.LCount) { + codePointsHangul = CharArray(Lexer.MAX_HANGUL_DECOMPOSITION_LENGTH) + codePointsHangul[readCodePoints++] = curSymb.toChar() + + curSymb = lexemes.currentChar + curSymbIndex = curSymb - Lexer.VBase + if (curSymbIndex >= 0 && curSymbIndex < Lexer.VCount) { + codePointsHangul[readCodePoints++] = curSymb.toChar() + lexemes.next() + curSymb = lexemes.currentChar + curSymbIndex = curSymb - Lexer.TBase + if (curSymbIndex >= 0 && curSymbIndex < Lexer.TCount) { + codePointsHangul[readCodePoints++] = curSymb.toChar() + lexemes.next() + + //LVT syllable + return HangulDecomposedCharSet(codePointsHangul, 3) + } else { + + //LV syllable + return HangulDecomposedCharSet(codePointsHangul, 2) + } + } else { + + //L jamo + return CharSet(codePointsHangul[0], hasFlag(CASE_INSENSITIVE)) + } + + /* + * We process single codepoint or decomposed codepoint. + * We collect decomposed codepoint and obtain + * one DecomposedCharSet. + */ + } else { + readCodePoints++ + + while (readCodePoints < Lexer.MAX_DECOMPOSITION_LENGTH + && !lexemes.isEmpty() && lexemes.isLetter() + && !Lexer.isDecomposedCharBoundary(lexemes.currentChar)) { + codePoints[readCodePoints++] = lexemes.next() + } + + /* + * We have read an ordinary symbol. + */ + if (readCodePoints == 1 && !Lexer.hasSingleCodepointDecomposition(codePoints[0])) { + return processCharSet(codePoints[0]) + } else { + return DecomposedCharSet(codePoints, readCodePoints) + } + } + } + + /** + * S->BS; S->QS; S->Q; B->a+ + */ + private fun processSubExpression(last: AbstractSet): AbstractSet { + var cur: AbstractSet + when { + lexemes.isLetter() && !lexemes.isNextSpecial && Lexer.isLetter(lexemes.lookAhead) -> { + when { + hasFlag(Pattern.CANON_EQ) -> { + cur = processDecomposedChar() + if (!lexemes.isEmpty() + && (lexemes.currentChar != Lexer.CHAR_RIGHT_PARENTHESIS || last is FinalSet) + && lexemes.currentChar != Lexer.CHAR_VERTICAL_BAR + && !lexemes.isLetter()) { + + cur = processQuantifier(last, cur) + } + } + lexemes.isHighSurrogate() || lexemes.isLowSurrogate() -> { + val term = processTerminal(last) // TODO: term is not null. + cur = processQuantifier(last, term) + } + else -> { + cur = processSequence() + } + } + } + lexemes.currentChar == Lexer.CHAR_RIGHT_PARENTHESIS -> { + if (last is FinalSet) { + throw PatternSyntaxException() + } + cur = EmptySet(last) + } + else -> { + val term = processTerminal(last) + cur = processQuantifier(last, term) + } + } + + if (!lexemes.isEmpty() + && (lexemes.currentChar != Lexer.CHAR_RIGHT_PARENTHESIS || last is FinalSet) + && lexemes.currentChar != Lexer.CHAR_VERTICAL_BAR) { + + val next = processSubExpression(last) + if (cur is LeafQuantifierSet + // '*' or '{0,}' quantifier + && cur.max == Quantifier.INF + && cur.min == 0 + && !next.first(cur.innerSet)) { + // An Optimizer node for the case where there is no intersection with the next node + cur = UnifiedQuantifierSet(cur) + } + cur.next = next + } else { + cur.next = last + } + return cur + } + + /** + * Q->T(*|+|?...) also do some optimizations. + */ + private fun processQuantifier(last: AbstractSet, term: AbstractSet): AbstractSet { + val quant = lexemes.currentChar + + if (term !is LeafSet) { + return when (quant) { + Lexer.QUANT_STAR, Lexer.QUANT_PLUS -> { + val q: QuantifierSet + + lexemes.next() + if (term.type == AbstractSet.TYPE_DOTSET) { + q = DotQuantifierSet(term, last, quant, AbstractLineTerminator.getInstance(flags), hasFlag(Pattern.DOTALL)) + } else { + q = GroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) + } + term.next = q + q + } + + Lexer.QUANT_ALT -> { + lexemes.next() + val q = GroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) + term.next = q + q + } + + Lexer.QUANT_STAR_R, Lexer.QUANT_PLUS_R, Lexer.QUANT_ALT_R -> { + lexemes.next() + val q = ReluctantGroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) + term.next = q + q + } + + Lexer.QUANT_PLUS_P, Lexer.QUANT_STAR_P, Lexer.QUANT_ALT_P -> { + lexemes.next() + PossessiveGroupQuantifierSet(Quantifier.fromLexerToken(quant), term, last, quant, groupQuantifierCount++) + } + + Lexer.QUANT_COMP -> { + val q = GroupQuantifierSet(lexemes.nextSpecial() as Quantifier, term, last, Lexer.QUANT_ALT, groupQuantifierCount++) + term.next = q + q + } + + Lexer.QUANT_COMP_R -> { + val q = ReluctantGroupQuantifierSet(lexemes.nextSpecial() as Quantifier, term, last, Lexer.QUANT_ALT, groupQuantifierCount++) + term.next = q + q + } + + Lexer.QUANT_COMP_P -> { + return PossessiveGroupQuantifierSet(lexemes.nextSpecial() as Quantifier, term, last, Lexer.QUANT_ALT, groupQuantifierCount++) + } + + else -> term + } + } else { + val leaf: LeafSet = term + return when (quant) { + Lexer.QUANT_STAR, Lexer.QUANT_PLUS -> { + lexemes.next() + val q = LeafQuantifierSet(Quantifier.fromLexerToken(quant), leaf, last, quant) + leaf.next = q + q + } + + Lexer.QUANT_STAR_R, Lexer.QUANT_PLUS_R -> { + lexemes.next() + val q = ReluctantLeafQuantifierSet(Quantifier.fromLexerToken(quant), leaf, last, quant) + leaf.next = q + q + } + + Lexer.QUANT_PLUS_P, Lexer.QUANT_STAR_P -> { + lexemes.next() + val q = PossessiveLeafQuantifierSet(Quantifier.fromLexerToken(quant), leaf, last, quant) + leaf.next = q + q + } + + Lexer.QUANT_ALT -> { + lexemes.next() + LeafQuantifierSet(Quantifier.altQuantifier, leaf, last, Lexer.QUANT_ALT) + } + + Lexer.QUANT_ALT_R -> { + lexemes.next() + ReluctantLeafQuantifierSet(Quantifier.altQuantifier, leaf, last, Lexer.QUANT_ALT_R) + } + + Lexer.QUANT_ALT_P -> { + lexemes.next() + PossessiveLeafQuantifierSet(Quantifier.altQuantifier, leaf, last, Lexer.QUANT_ALT_P) + } + + Lexer.QUANT_COMP -> { + LeafQuantifierSet(lexemes.nextSpecial() as Quantifier, leaf, last, Lexer.QUANT_COMP) + } + Lexer.QUANT_COMP_R -> { + ReluctantLeafQuantifierSet(lexemes.nextSpecial() as Quantifier, leaf, last, Lexer.QUANT_COMP_R) + } + Lexer.QUANT_COMP_P -> { + ReluctantLeafQuantifierSet(lexemes.nextSpecial() as Quantifier, leaf, last, Lexer.QUANT_COMP_P) + } + + else -> term + } + } + } + + /** + * T-> letter|[range]|{char-class}|(E) + */ + private fun processTerminal(last: AbstractSet): AbstractSet { + val term: AbstractSet + var char = lexemes.currentChar + // Process flags: (?...)(?...)... + while (char and 0xff00ffff.toInt() == Lexer.CHAR_FLAGS) { + lexemes.next() + flags = (char shr 16) and flagsBitMask + char = lexemes.currentChar + } + // The terminal is some kind of group: (E). Call processExpression for it. + if (char and 0x8000ffff.toInt() == Lexer.CHAR_LEFT_PARENTHESIS) { + lexemes.next() + var newFlags = flags + if (char and 0xff00ffff.toInt() == Lexer.CHAR_NONCAP_GROUP) { + newFlags = (char shr 16) and flagsBitMask + } + term = processExpression(char and 0xff00ffff.toInt(), newFlags, last) // Remove flags from the token. + if (lexemes.currentChar != Lexer.CHAR_RIGHT_PARENTHESIS) { + throw PatternSyntaxException() + } + lexemes.next() + } else { + // Other terminals. + when (char) { + Lexer.CHAR_LEFT_SQUARE_BRACKET -> { // Range: [...] + lexemes.next() + var negative = false + if (lexemes.currentChar == Lexer.CHAR_CARET) { + negative = true + lexemes.next() + } + + term = processRange(negative, last) + if (lexemes.currentChar != Lexer.CHAR_RIGHT_SQUARE_BRACKET) { + throw PatternSyntaxException() + } + // TODO: Set mode lexer as Mode.Range is set. + lexemes.setModeWithReread(Lexer.Mode.PATTERN) + lexemes.next() + } + + Lexer.CHAR_DOT -> { // Dot: . + lexemes.next() + term = DotSet(AbstractLineTerminator.getInstance(flags), hasFlag(DOTALL)) + } + + Lexer.CHAR_CARET -> { // Beginning of the string: ^ + lexemes.next() + term = SOLSet(AbstractLineTerminator.getInstance(flags), hasFlag(MULTILINE)) + consumersCount++ + } + + Lexer.CHAR_DOLLAR -> { // End of the string: $ + lexemes.next() + term = EOLSet(consumersCount++, AbstractLineTerminator.getInstance(flags), hasFlag(MULTILINE)) + + } + + // Word / non-word boundary. + Lexer.CHAR_WORD_BOUND -> { + lexemes.next() + term = WordBoundarySet(true) + } + + Lexer.CHAR_NONWORD_BOUND -> { + lexemes.next() + term = WordBoundarySet(false) + } + + Lexer.CHAR_END_OF_INPUT -> { // End of an input: \z + lexemes.next() + term = EOISet() + } + + Lexer.CHAR_END_OF_LINE -> { // End of a line: \Z + lexemes.next() + term = EOLSet(consumersCount++, AbstractLineTerminator.getInstance(flags)) + } + + Lexer.CHAR_START_OF_INPUT -> { // Start if an input: \A + lexemes.next() + term = SOLSet(AbstractLineTerminator.getInstance(flags)) + } + + Lexer.CHAR_PREVIOUS_MATCH -> { // A previous match: \G + lexemes.next() + term = PreviousMatchSet() + } + + // Back references: \1, \2 etc. + 0x80000000.toInt() or '1'.toInt(), + 0x80000000.toInt() or '2'.toInt(), + 0x80000000.toInt() or '3'.toInt(), + 0x80000000.toInt() or '4'.toInt(), + 0x80000000.toInt() or '5'.toInt(), + 0x80000000.toInt() or '6'.toInt(), + 0x80000000.toInt() or '7'.toInt(), + 0x80000000.toInt() or '8'.toInt(), + 0x80000000.toInt() or '9'.toInt() -> { + val number = (char and 0x7FFFFFFF) - '0'.toInt() + if (number < capturingGroupCount) { // All is ok - the group exists. + lexemes.next() + term = BackReferenceSet(number, consumersCount++, hasFlag(CASE_INSENSITIVE)) + // backRefs[number] is proved to be not null because the group is already created (number < capturingGroupCount) + backRefs[number]!!.isBackReferenced = true + needsBackRefReplacement = true // And process back references in the second pass. + } else { + throw PatternSyntaxException() // Wrong group number. + } + } + + // A special token (\D, \w etc), 'u0000' or the end of the pattern. + 0 -> { + val cc: AbstractCharClass? = lexemes.curSpecialToken as AbstractCharClass? + when { + cc != null -> { + term = processRangeSet(cc) + lexemes.next() + } + !lexemes.isEmpty() -> { + term = CharSet(char.toChar()) + lexemes.next() + } + else -> term = EmptySet(last) + } + } + + else -> { + when { + // A regular character. + char >= 0 && !lexemes.isSpecial -> { + term = processCharSet(char) + lexemes.next() + } + char == Lexer.CHAR_VERTICAL_BAR -> { + term = EmptySet(last) + } + char == Lexer.CHAR_RIGHT_PARENTHESIS -> { + if (last is FinalSet) { + throw PatternSyntaxException() + } + term = EmptySet(last) + } + else -> throw PatternSyntaxException() + } + } + } + } + return term + } + + + /** + * Process [...] ranges + */ + // TODO: We can merge it with processRangeExpression. + private fun processRange(negative: Boolean, last: AbstractSet): AbstractSet { + val res = processRangeExpression(negative) + val rangeSet = processRangeSet(res) + rangeSet.next = last + + return rangeSet + } + + private fun processRangeExpression(alt: Boolean): CharClass { + var result = CharClass(hasFlag(Pattern.CASE_INSENSITIVE), alt) + var buffer = -1 + var intersection = false + var firstInClass = true + + var notClosed = lexemes.currentChar != Lexer.CHAR_RIGHT_SQUARE_BRACKET + while (!lexemes.isEmpty() && (notClosed || firstInClass)) { + when (lexemes.currentChar) { + + Lexer.CHAR_RIGHT_SQUARE_BRACKET -> { + if (buffer >= 0) { + result.add(buffer) + } + buffer = ']'.toInt() + lexemes.next() + } + + Lexer.CHAR_LEFT_SQUARE_BRACKET -> { + if (buffer >= 0) { + result.add(buffer) + buffer = -1 + } + lexemes.next() + var negative = false + if (lexemes.currentChar == Lexer.CHAR_CARET) { + lexemes.next() + negative = true + } + + if (intersection) + result.intersection(processRangeExpression(negative)) + else + result.union(processRangeExpression(negative)) + intersection = false + lexemes.next() + } + + Lexer.CHAR_AMPERSAND -> { + if (buffer >= 0) { + result.add(buffer) + } + buffer = lexemes.next() // buffer == Lexer.CHAR_AMPERSAND since next() returns currentChar. + + /* + * If there is a start for subrange we will do an intersection + * otherwise treat '&' as a normal character + */ + if (lexemes.currentChar == Lexer.CHAR_AMPERSAND) { + if (lexemes.lookAhead == Lexer.CHAR_LEFT_SQUARE_BRACKET) { + lexemes.next() + intersection = true + buffer = -1 + } else { + lexemes.next() + if (firstInClass) { + // Skip "&&" at "[&&...]" or "[^&&...]" + result = processRangeExpression(false) + } else { + // Ignore "&&" at "[X&&]" ending where X != empty string + if (lexemes.currentChar != Lexer.CHAR_RIGHT_SQUARE_BRACKET) { + result.intersection(processRangeExpression(false)) + } + } + } + } else { + //treat '&' as a normal character + buffer = '&'.toInt() + } + } + + Lexer.CHAR_HYPHEN -> { + if (firstInClass + || lexemes.lookAhead == Lexer.CHAR_RIGHT_SQUARE_BRACKET + || lexemes.lookAhead == Lexer.CHAR_LEFT_SQUARE_BRACKET + || buffer < 0) { + // Treat the hypen as a normal character. + if (buffer >= 0) { + result.add(buffer) + } + buffer = '-'.toInt() + lexemes.next() + } else { + // A range. + lexemes.next() + var cur = lexemes.currentChar + + if (!lexemes.isSpecial + && (cur >= 0 + || lexemes.lookAhead == Lexer.CHAR_RIGHT_SQUARE_BRACKET + || lexemes.lookAhead == Lexer.CHAR_LEFT_SQUARE_BRACKET + || buffer < 0)) { + + try { + if (!Lexer.isLetter(cur)) { + cur = cur and 0xFFFF + } + result.add(buffer, cur) + } catch (e: Exception) { + throw PatternSyntaxException() + } + + lexemes.next() + buffer = -1 + } else { + throw PatternSyntaxException() + } + } + } + + Lexer.CHAR_CARET -> { + if (buffer >= 0) { + result.add(buffer) + } + buffer = '^'.toInt() + lexemes.next() + } + + 0 -> { + if (buffer >= 0) { + result.add(buffer) + } + val cs = lexemes.curSpecialToken as AbstractCharClass? + if (cs != null) { + result.add(cs) + buffer = -1 + } else { + buffer = 0 + } + + lexemes.next() + } + + else -> { + if (buffer >= 0) { + result.add(buffer) + } + buffer = lexemes.next() + } + } + + firstInClass = false + notClosed = lexemes.currentChar != Lexer.CHAR_RIGHT_SQUARE_BRACKET + } + if (notClosed) { + throw PatternSyntaxException() + } + if (buffer >= 0) { + result.add(buffer) + } + return result + } + + // TODO: Reread + private fun processRangeSet(charClass: AbstractCharClass): AbstractSet { + if (charClass.hasLowHighSurrogates()) { + val lowHighSurrRangeSet = SurrogateRangeSet(charClass.surrogates) + + if (charClass.mayContainSupplCodepoints) { + return CompositeRangeSet(SupplementaryRangeSet(charClass.withoutSurrogates, hasFlag(CASE_INSENSITIVE)), lowHighSurrRangeSet) + } + + return CompositeRangeSet(RangeSet(charClass.withoutSurrogates, hasFlag(CASE_INSENSITIVE)), lowHighSurrRangeSet) + } + + if (charClass.mayContainSupplCodepoints) { + return SupplementaryRangeSet(charClass, hasFlag(CASE_INSENSITIVE)) + } + + return RangeSet(charClass, hasFlag(CASE_INSENSITIVE)) + } + + private fun processCharSet(ch: Int): AbstractSet { + val isSupplCodePoint = Char.isSupplementaryCodePoint(ch) + + return when { + isSupplCodePoint -> SequenceSet(fromCharArray(Char.toChars(ch), 0, 2), hasFlag(CASE_INSENSITIVE)) + ch.toChar().isLowSurrogate() -> LowSurrogateCharSet(ch.toChar()) + ch.toChar().isHighSurrogate() -> HighSurrogateCharSet(ch.toChar()) + else -> CharSet(ch.toChar(), hasFlag(CASE_INSENSITIVE)) + } + } + + companion object { + //TODO: Use RegexOption enum here. + // Flags. + /** + * This constant specifies that a pattern matches Unix line endings ('\n') + * only against the '.', '^', and '$' meta characters. + */ + val UNIX_LINES = 1 shl 0 + + /** + * This constant specifies that a `Pattern` is matched + * case-insensitively. That is, the patterns "a+" and "A+" would both match + * the string "aAaAaA". + */ + val CASE_INSENSITIVE = 1 shl 1 + + /** + * This constant specifies that a `Pattern` may contain whitespace or + * comments. Otherwise comments and whitespace are taken as literal + * characters. + */ + val COMMENTS = 1 shl 2 + + /** + * This constant specifies that the meta characters '^' and '$' match only + * the beginning and end end of an input line, respectively. Normally, they + * match the beginning and the end of the complete input. + */ + val MULTILINE = 1 shl 3 + + /** + * This constant specifies that the whole `Pattern` is to be taken + * literally, that is, all meta characters lose their meanings. + */ + val LITERAL = 1 shl 4 + + /** + * This constant specifies that the '.' meta character matches arbitrary + * characters, including line endings, which is normally not the case. + */ + val DOTALL = 1 shl 5 + + /** + * This constant specifies that a character in a `Pattern` and a + * character in the input string only match if they are canonically + * equivalent. + */ + val CANON_EQ = 1 shl 6 + + /** Max number of back references supported. */ + internal val BACK_REF_NUMBER = 10 + + /** A bit mask that includes all defined match flags */ + internal val flagsBitMask = Pattern.UNIX_LINES or + Pattern.CASE_INSENSITIVE or + Pattern.COMMENTS or + Pattern.MULTILINE or + Pattern.LITERAL or + Pattern.DOTALL or + Pattern.CANON_EQ + + + /** + * Quotes a given string using "\Q" and "\E", so that all other meta-characters lose their special meaning. + * If the string is used for a `Pattern` afterwards, it can only be matched literally. + */ + // TODO: Replace with other functions. + fun quote(s: String): String { + return StringBuilder() + .append("\\Q") + .append(s.replace("\\E", "\\E\\\\E\\Q")) + .append("\\E").toString() + } + } +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/Quantifier.kt b/runtime/src/main/kotlin/kotlin/text/regex/Quantifier.kt new file mode 100644 index 00000000000..548d4096737 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/Quantifier.kt @@ -0,0 +1,41 @@ +// TODO: License. + +package kotlin.text + +import kotlin.IllegalArgumentException + +/** + * Represents RE quantifier; contains two fields responsible for min and max number of repetitions. + * Negative value for maximum number of repetition represents infinity(i.e. +,*) + + * @author Nikolay A. Kuznetsov + */ +// TODO: May be replace with some other class (Range?). +internal class Quantifier(val min: Int, val max: Int = min) : SpecialToken() { + + init { + if (min < 0 || max < -1) { + throw IllegalArgumentException() + } + } + + override fun toString() = "{$min, ${if (max == -1) "" else max}}" + + override val type: Type = SpecialToken.Type.QUANTIFIER + + companion object { + val starQuantifier = Quantifier(0, -1) + val plusQuantifier = Quantifier(1, -1) + val altQuantifier = Quantifier(0, 1) + + val INF = -1 + + fun fromLexerToken(token: Int) = when(token) { + Lexer.QUANT_STAR, Lexer.QUANT_STAR_P, Lexer.QUANT_STAR_R -> starQuantifier + Lexer.QUANT_ALT, Lexer.QUANT_ALT_P, Lexer.QUANT_ALT_R -> altQuantifier + Lexer.QUANT_PLUS, Lexer.QUANT_PLUS_P, Lexer.QUANT_PLUS_R -> plusQuantifier + else -> throw IllegalArgumentException() + } + } +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/Regex.kt b/runtime/src/main/kotlin/kotlin/text/regex/Regex.kt new file mode 100644 index 00000000000..4a6de641cae --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/Regex.kt @@ -0,0 +1,322 @@ +/* + * Copyright 2010-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +private interface FlagEnum { + val value: Int + val mask: Int +} + +private fun Iterable.toInt(): Int = this.fold(0, { value, option -> value or option.value }) + +private fun fromInt(value: Int): Set = + RegexOption.values().filterTo(mutableSetOf()) { value and it.mask == it.value } + +/** + * Provides enumeration values to use to set regular expression options. + */ +enum class RegexOption(override val value: Int, override val mask: Int = value) : FlagEnum { + // common + + /** Enables case-insensitive matching. Case comparison is Unicode-aware. */ + IGNORE_CASE(Pattern.CASE_INSENSITIVE), + + /** + * Enables multiline mode. + * In multiline mode the expressions `^` and `$` match just after or just before, + * respectively, a line terminator or the end of the input sequence. + */ + MULTILINE(Pattern.MULTILINE), + + //jvm-specific + + /** + * Enables literal parsing of the pattern. + * Metacharacters or escape sequences in the input sequence will be given no special meaning. + */ + LITERAL(Pattern.LITERAL), + + /** + * Enables Unix lines mode. + * In this mode, only the `'\n'` is recognized as a line terminator. + */ + UNIX_LINES(Pattern.UNIX_LINES), + + /** Permits whitespace and comments in pattern. */ + COMMENTS(Pattern.COMMENTS), + + /** Enables the mode, when the expression `.` matches any character, + * including a line terminator. + */ + DOT_MATCHES_ALL(Pattern.DOTALL), + + /** Enables equivalence by canonical decomposition. */ + CANON_EQ(Pattern.CANON_EQ) +} + + +/** + * Represents the results from a single capturing group within a [MatchResult] of [Regex]. + * + * @param value The value of captured group. + * @param range The range of indices in the input string where group was captured. + * + * The [range] property is available on JVM only. + */ +data class MatchGroup(val value: String, val range: IntRange) + +/** + * Represents an immutable regular expression. + * + * For pattern syntax reference see [Pattern] + */ +class Regex internal constructor(internal val nativePattern: Pattern) { + + enum class Mode { + FIND, MATCH + } + + /** Creates a regular expression from the specified [pattern] string and the default options. */ + constructor(pattern: String): this(Pattern(pattern)) + + /** Creates a regular expression from the specified [pattern] string and the specified single [option]. */ + constructor(pattern: String, option: RegexOption): this(Pattern(pattern, ensureUnicodeCase(option.value))) + + /** Creates a regular expression from the specified [pattern] string and the specified set of [options]. */ + constructor(pattern: String, options: Set): this(Pattern(pattern, ensureUnicodeCase(options.toInt()))) + + + /** The pattern string of this regular expression. */ + val pattern: String + get() = nativePattern.pattern + + private val startNode = nativePattern.startNode + + /** The set of options that were used to create this regular expression. */ + val options: Set = fromInt(nativePattern.flags) + + companion object { + /** Returns a literal regex for the specified [literal] string. */ + // TODO: Uncomment for native + fun fromLiteral(literal: String): Regex = Regex(literal, RegexOption.LITERAL) + + /** Returns a literal pattern for the specified [literal] string. */ + fun escape(literal: String): String = Pattern.quote(literal) + + /** + * Returns a replacement string for the given one that has all backslashes + * and dollar signs escaped. + */ + fun escapeReplacement(literal: String): String { + if (!literal.contains('\\') && !literal.contains('$')) + return literal + + val result = StringBuilder(literal.length * 2) + literal.forEach { + if (it == '\\' || it == '$') { + result.append('\\') + } + result.append(it) + } + + return result.toString() + } + + // TODO: Remove + private fun ensureUnicodeCase(flags: Int) = flags + } + + private fun doMatch(input: CharSequence, mode: Mode): MatchResult? { + // TODO: Harmony has a default constructor for MatchResult. Do we need it? + // TODO: Reuse the matchResult. + val matchResult = MatchResultImpl(input, this) + matchResult.mode = mode + val matches = startNode.matches(0, input, matchResult) >= 0 + if (!matches) { + return null + } + matchResult.finalizeMatch() + return matchResult + } + + /** Indicates whether the regular expression matches the entire [input]. */ + infix fun matches(input: CharSequence): Boolean = doMatch(input, Mode.MATCH) != null + + /** Indicates whether the regular expression can find at least one match in the specified [input]. */ + // TODO: Looks like we don't need Mode anymore. + fun containsMatchIn(input: CharSequence): Boolean = find(input) != null + + /** + * Returns the first match of a regular expression in the [input], beginning at the specified [startIndex]. + * + * @param startIndex An index to start search with, by default 0. Must be not less than zero and not greater than `input.length()` + * @return An instance of [MatchResult] if match was found or `null` otherwise. + */ + fun find(input: CharSequence, startIndex: Int = 0): MatchResult? { + if (startIndex < 0 || startIndex > input.length) { + throw IndexOutOfBoundsException() // TODO: Add a message. + } + // TODO: reuse the match result? + val matchResult = MatchResultImpl(input, this) + matchResult.mode = Mode.FIND + matchResult.startIndex = startIndex + val foundIndex = startNode.find(startIndex, input, matchResult) + if (foundIndex >= 0) { + matchResult.finalizeMatch() + return matchResult + } else { + /*matchResult.hitEnd = true + matchResult.startIndex = -1*/ + return null + } + } + + /** + * Returns a sequence of all occurrences of a regular expression within the [input] string, beginning at the specified [startIndex]. + */ + fun findAll(input: CharSequence, startIndex: Int = 0): Sequence + = generateSequence({ find(input, startIndex) }, MatchResult::next) + + /** + * Attempts to match the entire [input] CharSequence against the pattern. + * + * @return An instance of [MatchResult] if the entire input matches or `null` otherwise. + */ + fun matchEntire(input: CharSequence): MatchResult?= doMatch(input, Mode.MATCH) + + private fun processReplacement(match: MatchResult, replacement: String): String { + val result = StringBuilder(replacement.length) + var escaped = false + var backReference = false + for (ch in replacement) { + when { + escaped -> { + result.append(ch) + escaped = false + } + backReference -> { + if (ch !in '0'..'9') { + throw IllegalArgumentException("Incorrect back reference: $ch.") + } + val group = ch - '0' + result.append(match.groupValues[group]) + // We don't catch IndexOutOfBoundException here because + // it's a correct exception in case of a wrong group number. + // TODO: But we can rethrow it with more informative message. + backReference = false + } + ch == '\\' -> escaped = true + ch == '$' -> backReference = true + else -> result.append(ch) + } + } + if (backReference || escaped) { + throw IllegalArgumentException("Unexpected end of replacement.") + } + return result.toString() + } + + /** + * Replaces all occurrences of this regular expression in the specified [input] string with specified [replacement] expression. + * + * @param replacement A replacement expression that can include substitutions. See [Matcher.appendReplacement] for details. + */ + fun replace(input: CharSequence, replacement: String): String + = replace(input) { match -> processReplacement(match, replacement) } + + /** + * Replaces all occurrences of this regular expression in the specified [input] string with the result of + * the given function [transform] that takes [MatchResult] and returns a string to be used as a + * replacement for that match. + */ + fun replace(input: CharSequence, transform: (MatchResult) -> CharSequence): String { + var match: MatchResult? = find(input) ?: return input.toString() + + var lastStart = 0 + val length = input.length + val sb = StringBuilder(length) + do { + val foundMatch = match!! + sb.append(input, lastStart, foundMatch.range.start) + sb.append(transform(foundMatch)) + lastStart = foundMatch.range.endInclusive + 1 + match = foundMatch.next() + } while (lastStart < length && match != null) + + if (lastStart < length) { + sb.append(input, lastStart, length) + } + + return sb.toString() + } + + /** + * Replaces the first occurrence of this regular expression in the specified [input] string with specified [replacement] expression. + * + * @param replacement A replacement expression that can include substitutions. See [Matcher.appendReplacement] for details. + */ + fun replaceFirst(input: CharSequence, replacement: String): String + = replaceFirst(input) { match -> processReplacement(match, replacement) } + + /** + * Splits the [input] CharSequence around matches of this regular expression. + * + * @param limit Non-negative value specifying the maximum number of substrings the string can be split to. + * Zero by default means no limit is set. + */ + // TODO: replace all argument checks with require function. + fun split(input: CharSequence, limit: Int = 0): List { + require(limit >= 0, { "Limit must be non-negative, but was $limit." } ) + if (input.isEmpty()) { + return listOf("") + } else { + var lastStart = 0 + val result = mutableListOf() + var match: MatchResult? = find(input) + + while (match != null && (limit == 0 || result.size < limit - 1)) { + result.add(input.substring(lastStart, match.range.start)) + lastStart = match.range.endInclusive + 1 + match = match.next() + } + result.add(input.substring(lastStart, input.length)) + return result + } + } + + /** Returns the string representation of this regular expression, namely the [pattern] of this regular expression. */ + override fun toString(): String = nativePattern.toString() + + // Native specific ================================================================================================= + fun lookingAt(input: CharSequence): Boolean = doMatch(input, Mode.FIND) != null + + /** Indicates whether the regular expression can find at least one match in the specified [input] starting with [index]. */ + fun containsMatchIn(input: CharSequence, index: Int): Boolean = find(input, index) != null + + /** Replaces the first occurrence of this regular expression in the specified [input] string with specified using specified transfromation */ + fun replaceFirst(input: CharSequence, transform: (MatchResult) -> CharSequence): String { + val match = find(input) ?: return input.toString() + val length = input.length + val result = StringBuilder(length) + result.append(input, 0, match.range.start) + result.append(transform(match)) + if (match.range.endInclusive + 1 < length) { + result.append(input, match.range.endInclusive + 1, length) + } + return result.toString() + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/AbstractSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/AbstractSet.kt new file mode 100644 index 00000000000..a2eacdc40e5 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/AbstractSet.kt @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +import kotlin.AssertionError + +/** Basic class for sets which have no complex next node handling. */ +internal abstract class SimpleSet : AbstractSet { + override var next: AbstractSet = dummyNext + constructor() + constructor(type : Int): super(type) +} + +/** + * Basic class for nodes, representing given regular expression. + * Note: (Almost) All the classes representing nodes has 'set' suffix. + * + * @author Nikolay A. Kuznetsov + */ +internal abstract class AbstractSet(val type: Int = 0) { + + companion object { + const val TYPE_LEAF = 1 shl 0 + const val TYPE_FSET = 1 shl 1 + const val TYPE_QUANT = 1 shl 3 + const val TYPE_DOTSET = 0x80000000.toInt() or '.'.toInt() + + /** Counter for debugging purposes, represent unique node index. */ + var counter = 1 + + val dummyNext = object : AbstractSet() { + override var next: AbstractSet + get() = throw AssertionError("This method is not expected to be called.") + set(value) {} + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl) = + throw AssertionError("This method is not expected to be called.") + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = + throw AssertionError("This method is not expected to be called.") + override fun processSecondPassInternal(): AbstractSet = this + } + } + + var secondPassVisited = false + abstract var next: AbstractSet + + // These properties and toString() method are for debug purposes. + protected var debugIndex = AbstractSet.counter++.toString() + protected open val name: String + get() = "" + override fun toString(): String = "<$debugIndex:$name>" + + /** + * Checks if this node matches in given position and recursively call + * next node matches on positive self match. Returns positive integer if + * entire match succeed, negative otherwise. + * @param startIndex - string index to start from. + * @param testString - input string. + * @param matchResult - MatchResult to sore result into. + * @return -1 if match fails or n > 0; + */ + abstract fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int + + /** + * Attempts to apply pattern starting from this set/startIndex; returns + * index this search was started from, if value is negative, this means that + * this search didn't succeed, additional information could be obtained via + * matchResult. + * + * Note: this is default implementation for find method, it's based on + * matches, subclasses do not have to override find method unless + * more effective find method exists for a particular node type + * (sequence, i.e. substring, for example). Same applies for find back + * method. + * + * @param startIndex - starting index. + * @param testString - string to search in. + * @param matchResult - result of the match. + * @return last searched index. + */ + open fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int + = (startIndex..testString.length).firstOrNull { index -> matches(index, testString, matchResult) >= 0 } + ?: -1 + + /** + * @param leftLimit - an index, to finish search back (left limit, inclusive). + * @param rightLimit - an index to start search from (right limit, exclusive). // TODO: Is it really exclusive + * @param testString - test string. + * @param matchResult - match result. + * @return an index to start back search next time if this search fails(new left bound); + * if this search fails the value is negative. + */ + open fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int + = (rightLimit downTo leftLimit).firstOrNull { index -> matches(index, testString, matchResult) >= 0 } + ?: -1 + + /** + * Returns true, if this node has consumed any characters during + * positive match attempt, for example node representing character always + * consumes one character if it matches. If particular node matches + * empty sting this method will return false. + * + * @param matchResult - match result; + * @return true if the node consumes any character and false otherwise. + */ + abstract fun hasConsumed(matchResult: MatchResultImpl): Boolean + + /** + * Returns true if the given node intersects with this one, false otherwise. + * This method is being used for quantifiers construction, lets consider the + * following regular expression (a|b)*ccc. (a|b) does not intersects with "ccc" + * and thus can be quantified greedily (w/o kickbacks), like *+ instead of *. + + * @param set - A node the intersection is checked for. Usually a previous node. + * @return true if the given node intersects with this one, false otherwise. + */ + // TODO: May be rename. e.g. intersects for something else. + open fun first(set: AbstractSet): Boolean = true + + /** + * This method is used for replacement backreferenced sets. + * + * @return null if current node need not to be replaced, + * [JointSet] which is replacement of current node otherwise. + */ + open fun processBackRefReplacement(): JointSet? { + return null + } + + /** + * This method performs the second pass without checking if it's already performed or not. + */ + open fun processSecondPassInternal(): AbstractSet { + if (!next.secondPassVisited) { + this.next = next.processSecondPass() + } + return processBackRefReplacement() ?: this + } + + /** + * This method is used for traversing nodes after the first stage of compilation. + */ + open fun processSecondPass(): AbstractSet { + secondPassVisited = true + return processSecondPassInternal() + } +} \ No newline at end of file diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/AtomicJointSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/AtomicJointSet.kt new file mode 100644 index 00000000000..650cf8b3dcf --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/AtomicJointSet.kt @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * This class represent atomic group (?>X), once X matches, this match become unchangeable till the end of the match. + * + * @author Nikolay A. Kuznetsov + */ +open internal class AtomicJointSet(children: List, fSet: FSet) : NonCapturingJointSet(children, fSet) { + + /** Returns startIndex+shift, the next position to match */ + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val start = matchResult.getConsumed(groupIndex) + matchResult.setConsumed(groupIndex, startIndex) + children.forEach { + val shift = it.matches(startIndex, testString, matchResult) + if (shift >= 0) { + // AtomicFset always returns true, but saves the index to run this next.match() from; + // TODO: Try to get rid of this explicit cast to AtomicFSet + return next.matches((fSet as AtomicFSet).index, testString, matchResult) // TODO: We use next here. + } + } + + matchResult.setConsumed(groupIndex, start) + return -1 + } + + override val name: String + get() = "AtomicJointSet" + + // TODO: looks like we can replace it with just next property. + override var next: AbstractSet = dummyNext +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/BackReferenceSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/BackReferenceSet.kt new file mode 100644 index 00000000000..2970a13e448 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/BackReferenceSet.kt @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Case Insensitive back reference node; + * + * @author Nikolay A. Kuznetsov + */ +// TODO: rename the properties +open internal class BackReferenceSet(val referencedGroup: Int, val consCounter: Int, val ignoreCase: Boolean = false) + : SimpleSet() { + + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val groupValue = getReferencedGroupValue(matchResult) + + if (groupValue == null || startIndex + groupValue.length > testString.length) { + return -1 + } + + if (testString.startsWith(groupValue, startIndex, ignoreCase)) { + matchResult.setConsumed(consCounter, groupValue.length) + return next.matches(startIndex + groupValue.length, testString, matchResult) + } + return -1 + } + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val groupValue = getReferencedGroupValue(matchResult) + if (groupValue == null || startIndex + groupValue.length > testString.length) { + return -1 + } + + var index = startIndex + while (index <= testString.length) { + index = testString.indexOf(groupValue, index, ignoreCase) + if (index < 0) { + return -1 + } + if (index < testString.length + && next.matches(index + groupValue.length, testString, matchResult) >=0) { + return index + } + index++ + } + return -1 + } + + override fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val groupValue = getReferencedGroupValue(matchResult) + if (groupValue == null || leftLimit + groupValue.length > rightLimit) { + return -1 + } + + var index = rightLimit + while (index >= leftLimit) { + index = testString.lastIndexOf(groupValue, index, ignoreCase) + if (index < 0) { + return -1 + } + if (index >= 0 && next.matches(index + groupValue.length, testString, matchResult) >= 0) { + return index + } + index-- + } + return -1 + } + + + protected fun getReferencedGroupValue(matchResult: MatchResultImpl) = matchResult.group(referencedGroup) + override val name: String + get() = "back reference: $referencedGroup" + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + val result = matchResult.getConsumed(consCounter) != 0 + matchResult.setConsumed(consCounter, -1) + return result + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/CharSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/CharSet.kt new file mode 100644 index 00000000000..86f5a0200a0 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/CharSet.kt @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents node accepting single character. + * + * @author Nikolay A. Kuznetsov + */ +// TODO: Process next better - it is non-null in almost all cases. +open internal class CharSet(char: Char, val ignoreCase: Boolean = false) : LeafSet() { + + // We use only low case characters when working in case insensitive mode. + val char: Char = if (ignoreCase) char.toLowerCase() else char + + // Overrides ======================================================================================================= + + override fun accepts(startIndex: Int, testString: CharSequence): Int { + if (ignoreCase) { + return if (this.char == testString[startIndex].toLowerCase()) 1 else -1 + } else { + return if (this.char == testString[startIndex]) 1 else -1 + } + } + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + while (index < testString.length) { + index = testString.indexOf(char, index, ignoreCase) + if (index < 0) { + return -1 + } + if (next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index++ + } + return -1 + } + + override fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = rightLimit + while (index >= leftLimit) { + index = testString.lastIndexOf(char, index, ignoreCase) + if (index < 0) { + return -1 + } + if (next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index-- + } + return -1 + } + + override val name: String + get()= char.toString() + + override fun first(set: AbstractSet): Boolean { + if (ignoreCase) { + return super.first(set) // TODO: Add correct checks here. + } + return when (set) { + is CharSet -> set.char == char + is RangeSet -> set.accepts(0, char.toString()) > 0 + is SupplementaryCharSet -> false + is SupplementaryRangeSet -> set.contains(char) + else -> true + } + } +} \ No newline at end of file diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/CompositeRangeSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/CompositeRangeSet.kt new file mode 100644 index 00000000000..baf567a4f4e --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/CompositeRangeSet.kt @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Portions, Copyright © 1991-2005 Unicode, Inc. The following applies to Unicode. + * + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright © 1991-2005 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. Permission is + * hereby granted, free of charge, to any person obtaining a copy of the + * Unicode data files and any associated documentation (the "Data Files") + * or Unicode software and any associated documentation (the "Software") + * to deal in the Data Files or Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * and/or sell copies of the Data Files or Software, and to permit persons + * to whom the Data Files or Software are furnished to do so, provided that + * (a) the above copyright notice(s) and this permission notice appear with + * all copies of the Data Files or Software, (b) both the above copyright + * notice(s) and this permission notice appear in associated documentation, + * and (c) there is clear notice in each modified Data File or in the Software + * as well as in the documentation associated with the Data File(s) or Software + * that the data or software has been modified. + + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT + * OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + * + * 2. Additional terms from the Database: + * + * Copyright © 1995-1999 Unicode, Inc. All Rights reserved. + * + * Disclaimer + * + * The Unicode Character Database is provided as is by Unicode, Inc. + * No claims are made as to fitness for any particular purpose. No warranties + * of any kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been purchased + * on magnetic or optical media from Unicode, Inc., the sole remedy for any claim + * will be exchange of defective media within 90 days of receipt. This disclaimer + * is applicable for all other data files accompanying the Unicode Character Database, + * some of which have been compiled by the Unicode Consortium, and some of which + * have been supplied by other sources. + * + * Limitations on Rights to Redistribute This Data + * + * Recipient is granted the right to make copies in any form for internal + * distribution and to freely use the information supplied in the creation of + * products supporting the UnicodeTM Standard. The files in + * the Unicode Character Database can be redistributed to third parties or other + * organizations (whether for profit or not) as long as this notice and the disclaimer + * notice are retained. Information can be extracted from these files and used + * in documentation or programs, as long as there is an accompanying notice + * indicating the source. + */ + +package kotlin.text + +/** + * This class is used to split the range that contains surrogate characters into two ranges: + * the first consisting of these surrogate characters and the second consisting of all others characters + * from the parent range. This class represents the parent range split in such a manner. + */ +// TODO: May be we can merge sets with surrogates and without them. +// TODO: Use more specific constructor params (e.g. RangeSet) +internal class CompositeRangeSet(/* range without surrogates */ val withoutSurrogates: AbstractSet, + /* range containing surrogates only */ val surrogates: AbstractSet) : SimpleSet() { + + override var next: AbstractSet = dummyNext + get() = field + set(next) { + field = next + surrogates.next = next + withoutSurrogates.next = next + } + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var result = withoutSurrogates.matches(startIndex, testString, matchResult) + if (result < 0) { + result = surrogates.matches(startIndex, testString, matchResult) + } + return result + } + + override val name: String + get() = "CompositeRangeSet: " + " " + withoutSurrogates + " " + surrogates + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true + + override fun first(set: AbstractSet): Boolean { + return true + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt new file mode 100644 index 00000000000..d1e3bd920b2 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/DecomposedCharSet.kt @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** Represents canonical decomposition of Unicode character. Is used when CANON_EQ flag of Pattern class is specified. */ +// TODO: Refactor it. +open internal class DecomposedCharSet( + /** Decomposition of the Unicode codepoint */ + private val decomposedChar: IntArray, + /** Length of useful part of decomposedChar decomposedCharLength <= decomposedChar.length */ + private val decomposedCharLength: Int) : SimpleSet() { + + /** Contains information about number of chars that were read for a codepoint last time */ + private var readCharsForCodePoint = 1 + + /** UTF-16 encoding of decomposedChar */ + private val decomposedCharUTF16: String by lazy { + val strBuff = StringBuilder() + + for (i in 0..decomposedCharLength - 1) { + strBuff.append(Char.toChars(decomposedChar[i])) + } + return@lazy strBuff.toString() + } + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var strIndex = startIndex + val rightBound = testString.length + + if (strIndex >= rightBound) { + return -1 + } + + // We read testString and decompose it gradually to compare with this decomposedChar at position strIndex + var curChar = codePointAt(strIndex, testString, rightBound) + strIndex += readCharsForCodePoint + var decomposedCurrentCodePoint: IntArray? = Lexer.getDecomposition(curChar) + var readCodePoints = 0 + var i = 0 + // All decompositions have length that is less or equal Lexer.MAX_DECOMPOSITION_LENGTH + var decomposedCodePoint: IntArray + if (decomposedCurrentCodePoint == null) { + decomposedCodePoint = IntArray(Lexer.MAX_DECOMPOSITION_LENGTH) + decomposedCodePoint[readCodePoints++] = curChar + } else { + i = decomposedCurrentCodePoint.size + decomposedCodePoint = decomposedCurrentCodePoint.copyOf(Lexer.MAX_DECOMPOSITION_LENGTH) + readCodePoints += i + } + + if (strIndex < rightBound) { + curChar = codePointAt(strIndex, testString, rightBound) + + // Read testString until we met a decomposed char boundary and decompose obtained portion of testString. + while (readCodePoints < Lexer.MAX_DECOMPOSITION_LENGTH && !Lexer.isDecomposedCharBoundary(curChar)) { + + if (!Lexer.hasDecompositionNonNullCanClass(curChar)) { + decomposedCodePoint[readCodePoints++] = curChar + } else { + + /* + * A few codepoints have decompositions and non null + * canonical classes, we have to take them into + * consideration, but general rule is: + * if canonical class != 0 then no decomposition + */ + decomposedCurrentCodePoint = Lexer.getDecomposition(curChar) + + /* + * Length of such decomposition is 1 or 2. See UnicodeData file + * http://www.unicode.org/Public/4.0-Update/UnicodeData-4.0.0.txt + */ + // hasDecompositionNonNullCanClass(curChar) == true, so decomposedCurrentCodePoint != null. + if (decomposedCurrentCodePoint!!.size == 2) { + decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0] + decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[1] + } else { + decomposedCodePoint[readCodePoints++] = decomposedCurrentCodePoint[0] + } + } + + strIndex += readCharsForCodePoint + + if (strIndex < rightBound) { + curChar = codePointAt(strIndex, testString, rightBound) + } else { + break + } + } + } + + // Some optimization since length of decomposed char is <= 3 usually + when (readCodePoints) { + 0, 1, 2 -> {} + + 3 -> { + var i1 = Lexer.getCanonicalClass(decomposedCodePoint[1]) + val i2 = Lexer.getCanonicalClass(decomposedCodePoint[2]) + + if (i2 != 0 && i1 > i2) { + i1 = decomposedCodePoint[1] + decomposedCodePoint[1] = decomposedCodePoint[2] + decomposedCodePoint[2] = i1 + } + } + + else -> decomposedCodePoint = Lexer.getCanonicalOrder(decomposedCodePoint, readCodePoints) + } + + // Compare decomposedChar with decomposed char that was just read from testString + if (readCodePoints != decomposedCharLength) { + return -1 + } + + if ((0 until readCodePoints).firstOrNull { decomposedCodePoint[i] != decomposedChar[i] } != null) { + return -1 + } + return next.matches(strIndex, testString, matchResult) + } + + override val name: String + get() = "decomposed char: $decomposedChar" + + /** Reads Unicode codepoint from [testString] starting from [strIndex] until [rightBound]. */ + fun codePointAt(strIndex: Int, testString: CharSequence, rightBound: Int): Int { + var index = strIndex + + // We store information about number of codepoints we read at variable readCharsForCodePoint. + val curChar: Int + readCharsForCodePoint = 1 + if (index < rightBound - 1) { + val high = testString[index++] + val low = testString[index] + + if (Char.isSurrogatePair(high, low)) { + curChar = Char.toCodePoint(high, low) + readCharsForCodePoint = 2 + } else { + curChar = high.toInt() + } + } else { + curChar = testString[index].toInt() + } + + return curChar + } + + override fun first(set: AbstractSet): Boolean { + return if (set is DecomposedCharSet) + set.decomposedChar.contentEquals(decomposedChar) + else + true + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/DotQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/DotQuantifierSet.kt new file mode 100644 index 00000000000..4e0c827eed4 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/DotQuantifierSet.kt @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Special node for ".*" construction. + * The main idea here is to find line terminator and try to find the rest of the construction from this point. + * @author Nikolay A. Kuznetsov + */ +// TODO: Add optimized implementation for '.+' case +internal class DotQuantifierSet( + innerSet: AbstractSet, + next: AbstractSet, + type: Int, + val lineTerminator: AbstractLineTerminator, + val matchLineTerminator: Boolean = false +) : QuantifierSet(innerSet, next, type) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val rightBound = testString.length + val startSearch = if (matchLineTerminator) rightBound else testString.findLineTerminator(startIndex, rightBound) + + if (startSearch <= startIndex) { + return if (type.toChar() == '+') { + -1 + } else { + next.matches(startIndex, testString, matchResult) + } + } + val result = next.findBack(startIndex, startSearch, testString, matchResult) + if (type.toChar() == '+' && result == startIndex) { + return -1 + } + return result + } + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val rightBound = testString.length + if (matchLineTerminator) { + val foundIndex = next.findBack(startIndex, rightBound, testString, matchResult) + if (foundIndex >= 0 && !(type.toChar() == '+' && foundIndex == startIndex)) { + return startIndex + } else { + return -1 + } + } else { + // 1. find first occurrence of the searched pattern. + var nextFound = next.find(startIndex, testString, matchResult) + if (nextFound < 0) { + return -1 + } + + // 2. Check if we have other occurrences till the end of line (because .* is greedy and we need the last one). + val nextFoundLast = next.findBack(nextFound, + testString.findLineTerminator(nextFound, rightBound), + testString, matchResult) + nextFound = maxOf(nextFound, nextFoundLast) + + // 3. Find the left boundary of this search. + val leftBound = findBackLineTerminator(startIndex, nextFound, testString) + if (type.toChar() == '+' && leftBound + 1 == nextFound) { + return -1 + } + return leftBound + 1 + } + } + + /** + * Find the first line terminator between [from] (inclusive) and [to] (exclusive) indices. + * Returns [to] if no terminator found. + */ + private fun CharSequence.findLineTerminator(from: Int, to: Int): Int = + (from until to).firstOrNull { lineTerminator.isLineTerminator(this[it]) } ?: to + + /** + * Find the first line terminator between [from] (inclusive) and [to] (exclusive) indices. + * Returns [from - 1] if no terminator found. + */ + private fun findBackLineTerminator(from: Int, to: Int, testString: CharSequence): Int = + (from until to).lastOrNull { lineTerminator.isLineTerminator(testString[it]) } ?: from - 1 + + override val name: String + get() = ".*" +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/DotSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/DotSet.kt new file mode 100644 index 00000000000..5eced30509f --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/DotSet.kt @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Node accepting any character except line terminators. + * + * @author Nikolay A. Kuznetsov + */ +// TODO: Looks like we can extend AbstractSet instead of JointSet here. +internal class DotSet(val lt: AbstractLineTerminator, val matchLineTerminator: Boolean) + : SimpleSet(AbstractSet.TYPE_DOTSET) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val rightBound = testString.length + if (startIndex >= rightBound) { + return -1 + } + + val high = testString[startIndex] + if (high.isHighSurrogate() && startIndex + 2 <= rightBound) { + val low = testString[startIndex + 1] + if (Char.isSurrogatePair(high, low)) { + if (!matchLineTerminator && lt.isLineTerminator(Char.toCodePoint(high, low))) { + return -1 + } else { + return next.matches(startIndex + 2, testString, matchResult) + } + } + } + if (!matchLineTerminator && lt.isLineTerminator(high)) { + return -1 + } else { + return next.matches(startIndex + 1, testString, matchResult) + } + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true + override val name: String + get() = "." +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/EOISet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/EOISet.kt new file mode 100644 index 00000000000..3cf531a7525 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/EOISet.kt @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents end of input '\z', i.e. matches only character after the last one; + * + * @author Nikolay A. Kuznetsov + */ +internal class EOISet : SimpleSet() { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + if (startIndex < testString.length) { + return -1 + } + return next.matches(startIndex, testString, matchResult) + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = false + override val name: String + get() = "EOI" +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/EOLSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/EOLSet.kt new file mode 100644 index 00000000000..9828039b472 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/EOLSet.kt @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents a node for a '$' sign. + * Note: In Kotlin we use only the "anchoring bounds" mode when "$" matches the end of a match region. + * See: http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#useAnchoringBounds-boolean- + * + * @author Nikolay A. Kuznetsov + */ +// TODO: What is consCounter? +internal class EOLSet(val consCounter: Int, val lt: AbstractLineTerminator, val multiline: Boolean = false) : SimpleSet() { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val rightBound = testString.length + val remainingChars = rightBound - startIndex + + when { + startIndex >= rightBound || + remainingChars == 1 && lt.isLineTerminator(testString[startIndex]) || + remainingChars == 2 && lt.isLineTerminatorPair(testString[startIndex], testString[startIndex+1]) || + multiline && lt.isLineTerminator(testString[startIndex]) -> { + matchResult.setConsumed(consCounter, 0) + return next.matches(startIndex, testString, matchResult) + } + } + return -1 + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + val result = matchResult.getConsumed(consCounter) != 0 + matchResult.setConsumed(consCounter, -1) + return result + } + + override val name: String + get()= "" +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/EmptySet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/EmptySet.kt new file mode 100644 index 00000000000..57ebf13691e --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/EmptySet.kt @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Valid constant zero character match. + * + * @author Nikolay A. Kuznetsov + */ +internal class EmptySet(override var next: AbstractSet) : LeafSet() { + + override val charCount = 0 + + override fun accepts(startIndex: Int, testString: CharSequence): Int = 0 + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + for (index in startIndex..testString.length) { + if (index < testString.length) { + if (testString[index].isLowSurrogate() && + index > 0 && testString[index - 1].isHighSurrogate()) { + continue + } + } + if (next.matches(index, testString, matchResult) >= 0) { + return index + } + } + return -1 + } + + override fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + for (index in rightLimit downTo leftLimit) { + if (index < testString.length) { + if (testString[index].isLowSurrogate() && + index > 0 && testString[index - 1].isHighSurrogate()) { + continue + } + } + if (next.matches(index, testString, matchResult) >= 0) { + return index + } + } + return -1 + } + + + override val name: String + get()= "" + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return false + } + +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/FSets.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/FSets.kt new file mode 100644 index 00000000000..132eb77ac58 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/FSets.kt @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * The node which marks end of the particular group. + * @author Nikolay A. Kuznetsov + */ +// TODO: Rename? +open internal class FSet(val groupIndex: Int) : SimpleSet() { + + var isBackReferenced = false + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val oldEnd = matchResult.getEnd(groupIndex) + matchResult.setEnd(groupIndex, startIndex) + val shift = next.matches(startIndex, testString, matchResult) + if (shift < 0) { + matchResult.setEnd(groupIndex, oldEnd) + } + return shift + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = false + override val name: String + get() = "fSet" + + override fun processSecondPass(): FSet { + val result = super.processSecondPass() + assert(result == this) + return this + } + + /** + * Marks the end of the particular group and not take into account possible + * kickbacks (required for atomic groups, for instance) + */ + internal class PossessiveFSet : SimpleSet() { + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + return startIndex + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return false + } + + override val name: String + get() = "possessiveFSet" + } + + companion object { + var possessiveFSet = PossessiveFSet() + } +} + +/** + * Special construction which marks end of pattern. + * + * @author Nikolay A. Kuznetsov + */ +internal class FinalSet : FSet(0) { + + override fun matches(startIndex: Int, testString: CharSequence, + matchResult: MatchResultImpl): Int { + if (matchResult.mode == Regex.Mode.FIND || startIndex == testString.length) { + matchResult.setEnd(0, startIndex) + return startIndex + } + return -1 + } + + override val name: String + get() = "FinalSet" +} + +/** + * Non-capturing group closing node. + * + * @author Nikolay A. Kuznetsov + */ +internal class NonCapFSet(groupIndex: Int) : FSet(groupIndex) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + matchResult.setConsumed(groupIndex, startIndex - matchResult.getConsumed(groupIndex)) // TODO: Don't understand it. + return next.matches(startIndex, testString, matchResult) + } + + override val name: String + get() = "NonCapFSet" + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return false + } +} + +/** + * LookAhead FSet, always returns true + * @author Nikolay A. Kuznetsov + */ +internal class AheadFSet : FSet(-1) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + return startIndex + } + + override val name: String + get() = "AheadFSet" +} + +/** + * FSet for lookbehind constructs. Checks if string index saved by corresponding + * jointSet in "consumers" equals to current index and return current string + * index, return -1 otherwise. + + * @author Nikolay A. Kuznetsov + */ +internal class BehindFSet(groupIndex: Int) : FSet(groupIndex) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val rightBound = matchResult.getConsumed(groupIndex) + return if (rightBound == startIndex) startIndex else -1 + } + + override val name: String + get() = "BehindFSet" +} + +/** + * Represents an end of an atomic group. + * + * @author Nikolay A. Kuznetsov + */ +internal class AtomicFSet(groupIndex: Int) : FSet(groupIndex) { + + var index: Int = 0 + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + matchResult.setConsumed(groupIndex, startIndex - matchResult.getConsumed(groupIndex)) + index = startIndex + return startIndex + } + + override val name: String + get() = "AtomicFSet" + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return false + } +} \ No newline at end of file diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/GroupQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/GroupQuantifierSet.kt new file mode 100644 index 00000000000..cb89590126c --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/GroupQuantifierSet.kt @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Default quantifier over groups, in fact this type of quantifier is + * generally used for constructions we cant identify number of characters they + * consume. + + * @author Nikolay A. Kuznetsov + */ +open internal class GroupQuantifierSet( + val quantifier: Quantifier, + innerSet: AbstractSet, + next: AbstractSet, + type: Int, + val groupQuantifierIndex: Int // It's used to remember a number of the innerSet occurrences during the recursive search. +) : QuantifierSet(innerSet, next, type) { + + val max: Int get() = quantifier.max + val min: Int get() = quantifier.min + + // We call innerSet.matches here, if it succeeds it call next.matches where next is this QuantifierSet. + // So we have a recursive searching procedure. + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + + if (!innerSet.hasConsumed(matchResult)) { + return next.matches(startIndex, testString, matchResult) + } + + // TODO: We can store an optimized functions for specific cases (like *) in a separate callable reference and + // TODO: call it instead of checks during matching. + // Fast case: '*' or {0, } - no need to count occurrences. + if (min == 0 && max == Quantifier.INF) { + val nextIndex = innerSet.matches(startIndex, testString, matchResult) + + return if (nextIndex < 0) { + next.matches(startIndex, testString, matchResult) + } else { + nextIndex + } + } + + val enterCount = matchResult.enterCounters[groupQuantifierIndex] + + // can't go inner set; + if (max != Quantifier.INF && enterCount >= max) { + return next.matches(startIndex, testString, matchResult) + } + + // go inner set; + matchResult.enterCounters[groupQuantifierIndex]++ + val nextIndex = innerSet.matches(startIndex, testString, matchResult) + + if (nextIndex < 0) { + matchResult.enterCounters[groupQuantifierIndex]-- + if (enterCount >= min) { + return next.matches(startIndex, testString, matchResult) + } else { + matchResult.enterCounters[groupQuantifierIndex] = 0 + return -1 + } + } else { + matchResult.enterCounters[groupQuantifierIndex] = 0 + return nextIndex + } + + } + + override val name: String + get() = quantifier.toString() +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/HangulDecomposedCharSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/HangulDecomposedCharSet.kt new file mode 100644 index 00000000000..335bc519948 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/HangulDecomposedCharSet.kt @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents canonical decomposition of Hangul syllable. Is used when + * CANON_EQ flag of Pattern class is specified. + */ +// TODO: Refactor it. +internal class HangulDecomposedCharSet( + /** + * Decomposed Hangul syllable. + */ + private val decomposedChar: CharArray, + /** + * Length of useful part of decomposedChar + * decomposedCharLength <= decomposedChar.length + */ + private val decomposedCharLength: Int) : SimpleSet() { + + /** + * String representing syllable + */ + private val decomposedCharUTF16: String by lazy { + fromCharArray(decomposedChar, 0, decomposedChar.size) + } + + override val name: String + get() = "decomposed Hangul syllable: $decomposedCharUTF16" + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + + /* + * All decompositions for Hangul syllables have length that + * is less or equal Lexer.MAX_DECOMPOSITION_LENGTH + */ + val rightBound = testString.length + var SyllIndex = 0 + val decompSyllable = IntArray(Lexer + .MAX_HANGUL_DECOMPOSITION_LENGTH) + val decompCurSymb: IntArray? + var curSymb: Char + + /* + * For details about Hangul composition and decomposition see + * http://www.unicode.org/versions/Unicode4.0.0/ch03.pdf + * "3.12 Conjoining Jamo Behavior" + */ + var LIndex = -1 + var VIndex = -1 + var TIndex = -1 + + if (index >= rightBound) { + return -1 + } + curSymb = testString[index++] + decompCurSymb = Lexer.getHangulDecomposition(curSymb.toInt()) + + if (decompCurSymb == null) { + + /* + * We deal with ordinary letter or sequence of jamos + * at index at testString. + */ + decompSyllable[SyllIndex++] = curSymb.toInt() + LIndex = curSymb.toInt() - Lexer.LBase + + if (LIndex < 0 || LIndex >= Lexer.LCount) { + + /* + * Ordinary letter, that doesn't match this + */ + return -1 + } + + if (index < rightBound) { + curSymb = testString[index] + VIndex = curSymb.toInt() - Lexer.VBase + } + + if (VIndex < 0 || VIndex >= Lexer.VCount) { + + /* + * Single L jamo doesn't compose Hangul syllable, + * so doesn't match + */ + return -1 + } + index++ + decompSyllable[SyllIndex++] = curSymb.toInt() + + if (index < rightBound) { + curSymb = testString[index] + TIndex = curSymb.toInt() - Lexer.TBase + } + + if (TIndex < 0 || TIndex >= Lexer.TCount) { + + /* + * We deal with LV syllable at testString, so + * compare it to this + */ + return if (decomposedCharLength == 2 + && decompSyllable[0] == decomposedChar[0].toInt() + && decompSyllable[1] == decomposedChar[1].toInt()) + next.matches(index, testString, matchResult) + else + -1 + } + index++ + decompSyllable[SyllIndex++] = curSymb.toInt() + + /* + * We deal with LVT syllable at testString, so + * compare it to this + */ + return if (decomposedCharLength == 3 + && decompSyllable[0] == decomposedChar[0].toInt() + && decompSyllable[1] == decomposedChar[1].toInt() + && decompSyllable[2] == decomposedChar[2].toInt()) + next.matches(index, testString, matchResult) + else + -1 + } else { + + /* + * We deal with Hangul syllable at index at testString. + * So we decomposed it to compare with this. + */ + var i = 0 + + if (decompCurSymb.size != decomposedCharLength) { + return -1 + } + + while (i < decomposedCharLength) { + if (decompCurSymb[i] != decomposedChar[i].toInt()) { + return -1 + } + i++ + } + return next.matches(index, testString, matchResult) + } + } + + override fun first(set: AbstractSet): Boolean { + return if (set is HangulDecomposedCharSet) + set.decomposedCharUTF16 == decomposedCharUTF16 + else + true + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return true + } +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/JointSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/JointSet.kt new file mode 100644 index 00000000000..55693d88df1 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/JointSet.kt @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents group, which is alternation of other subexpression. + * One should think about "group" in this model as JointSet opening group and corresponding FSet closing group. + * + * @author Nikolay A. Kuznetsov + */ +open internal class JointSet(children: List, fSet: FSet) : AbstractSet() { + + protected var children: MutableList = mutableListOf().apply { addAll(children) } + + var fSet: FSet = fSet + protected set + + var groupIndex: Int = fSet.groupIndex + protected set + + /** + * Returns startIndex+shift, the next position to match + */ + // TODO: Why don't we use a next here? + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + if (children.isEmpty()) { + return -1 + } + val oldStart = matchResult.getStart(groupIndex) + matchResult.setStart(groupIndex, startIndex) + children.forEach { + val shift = it.matches(startIndex, testString, matchResult) + if (shift >= 0) { + return shift + } + } + matchResult.setStart(groupIndex, oldStart) + return -1 + } + + override var next: AbstractSet + get() = fSet.next + set(next) { + fSet.next = next + } + + override val name: String + get() = "JointSet" + override fun first(set: AbstractSet): Boolean = children.any { it.first(set) } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return !(matchResult.getEnd(groupIndex) >= 0 && matchResult.getStart(groupIndex) == matchResult.getEnd(groupIndex)) + } + + override fun processSecondPassInternal(): AbstractSet { + val fSet = this.fSet + if (!fSet.secondPassVisited) { + val newFSet = fSet.processSecondPass() + assert(newFSet == fSet) + } + + children.replaceAll { child -> if (!child.secondPassVisited) child.processSecondPass() else child } + return super.processSecondPassInternal() + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt new file mode 100644 index 00000000000..7dbfa048f6e --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafQuantifierSet.kt @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +import kotlin.RuntimeException + +/** + * Generalized greedy quantifier node over the leaf nodes. + * - a{n,m}; + * - a* == a{0, }; + * - a? == a{0, 1}; + * - a+ == a{1, }; + * + * @author Nikolay A. Kuznetsov + */ +open internal class LeafQuantifierSet(var quantifier: Quantifier, + innerSet: LeafSet, + next: AbstractSet, + type: Int +) : QuantifierSet(innerSet, next, type) { + + val leaf: LeafSet get() = super.innerSet as LeafSet + val min: Int get() = quantifier.min + val max: Int get() = quantifier.max + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + var occurrences = 0 + + // Process first occurrences of the sequence being looked for. + while (occurrences < min) { + if (index + leaf.charCount > testString.length) { + return -1 + } + + val shift = leaf.accepts(index, testString) + if (shift < 1) { + return -1 + } + index += shift + occurrences++ + } + + // Process occurrences between min and max. + while ((max < 0 || occurrences < max) && index + leaf.charCount <= testString.length) { + val shift = leaf.accepts(index, testString) + if (shift < 1) { + break + } + index += shift + occurrences++ + } + + // Roll back if the next node does't match the remaining string. + while (occurrences >= min) { + val shift = next.matches(index, testString, matchResult) + if (shift >= 0) { + return shift + } + index -= leaf.charCount + occurrences-- + } + return -1 + } + + override val name: String + get() = quantifier.toString() + + override var innerSet: AbstractSet + get() = super.innerSet + set(innerSet) { + if (innerSet !is LeafSet) + throw RuntimeException() + super.innerSet = innerSet + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafSet.kt new file mode 100644 index 00000000000..25858566d86 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/LeafSet.kt @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Base class for nodes representing leaf tokens of the RE, those who consumes fixed number of characters. + * + * @author Nikolay A. Kuznetsov + */ +internal abstract class LeafSet : SimpleSet(AbstractSet.TYPE_LEAF) { + + open val charCount = 1 + + /** + * Returns "shift", the number of accepted chars. + * Commonly internal function, but called by quantifiers. + */ + // TODO: Magic function. Use by another way? + abstract fun accepts(startIndex: Int, testString: CharSequence): Int + + /** + * Checks if we can enter this state and pass the control to the next one. + * Return positive value if match succeeds, negative otherwise. + */ + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + if (startIndex + charCount > testString.length) { + return -1 + } + + val shift = accepts(startIndex, testString) // TODO: may be move the check above in accept function. + if (shift < 0) { + return -1 + } + + return next.matches(startIndex + shift, testString, matchResult) + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return true + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/LookAheadSets.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/LookAheadSets.kt new file mode 100644 index 00000000000..b2e31090246 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/LookAheadSets.kt @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Positive lookahead node. + * + * @author Nikolay A. Kuznetsov + */ +internal class PositiveLookAheadSet(children: List, fSet: FSet) : AtomicJointSet(children, fSet) { + + /** Returns startIndex+shift, the next position to match */ + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + children.forEach { + val shift = it.matches(startIndex, testString, matchResult) + if (shift >= 0) { + // PosLookaheadFset always returns true, position remains the same next.match() from; + return next.matches(startIndex, testString, matchResult) + } + } + return -1 + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true // TODO: May be false? + override val name: String + get() = "PositiveLookaheadJointSet" +} + +/** + * Negative look ahead node. + * + * @author Nikolay A. Kuznetsov + */ +internal class NegativeLookAheadSet(children: List, fSet: FSet) : AtomicJointSet(children, fSet) { + + /** Returns startIndex+shift, the next position to match */ + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + children.forEach { + if (it.matches(startIndex, testString, matchResult) >= 0) { + return -1 + } + } + + return next.matches(startIndex, testString, matchResult) + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true + override val name: String + get() = "NegativeLookaheadJointSet" +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/LookBehindSets.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/LookBehindSets.kt new file mode 100644 index 00000000000..4c2925f9527 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/LookBehindSets.kt @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Positive lookbehind node. + * + * @author Nikolay A. Kuznetsov + */ +internal class PositiveLookBehindSet(children: List, fSet: FSet) : AtomicJointSet(children, fSet) { + + /** Returns startIndex+shift, the next position to match */ + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + + matchResult.setConsumed(groupIndex, startIndex) + children.forEach { + if (it.findBack(0, startIndex, testString, matchResult) >= 0) { + matchResult.setConsumed(groupIndex, -1) + return next.matches(startIndex, testString, matchResult) + } + } + + return -1 + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true // TODO: False was here. It's true just for experiment. + override val name: String + get() = "PositiveBehindJointSet" +} + +/** + * Negative look behind node. + * + * @author Nikolay A. Kuznetsov + */ +internal class NegativeLookBehindSet(children: List, fSet: FSet) : AtomicJointSet(children, fSet) { + + /** Returns startIndex+shift, the next position to match */ + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + matchResult.setConsumed(groupIndex, startIndex) + + children.forEach { + val shift = it.findBack(0, startIndex, testString, matchResult) + if (shift >= 0) { + return -1 + } + } + + return next.matches(startIndex, testString, matchResult) + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true + override val name: String + get() = "NegativeBehindJointSet" +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/NonCapturingJointSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/NonCapturingJointSet.kt new file mode 100644 index 00000000000..a2bc55d75c8 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/NonCapturingJointSet.kt @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Node representing non-capturing group + * @author Nikolay A. Kuznetsov + */ +open internal class NonCapturingJointSet(children: List, fSet: FSet) : JointSet(children, fSet) { + + /** + * Returns startIndex+shift, the next position to match + */ + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val start = matchResult.getConsumed(groupIndex) + matchResult.setConsumed(groupIndex, startIndex) + + children.forEach { + val shift = it.matches(startIndex, testString, matchResult) + if (shift >= 0) { + return shift + } + } + + matchResult.setConsumed(groupIndex, start) + return -1 + } + + override val name: String + get() = "NonCapturingJointSet" + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean { + return matchResult.getConsumed(groupIndex) != 0 + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveGroupQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveGroupQuantifierSet.kt new file mode 100644 index 00000000000..b286be02cb9 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveGroupQuantifierSet.kt @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Possessive quantifier set over groups. + * + * @author Nikolay A. Kuznetsov + */ +internal class PossessiveGroupQuantifierSet( + quantifier: Quantifier, + innerSet: AbstractSet, + next: AbstractSet, + type: Int, + setCounter: Int +): GroupQuantifierSet(quantifier, innerSet, next, type, setCounter) { + + init { + innerSet.next = FSet.possessiveFSet // TODO: Try to use such an approach for other sets. + } + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + + var index = startIndex + var nextIndex: Int = innerSet.matches(index, testString, matchResult) + var occurrences = 0 + while (nextIndex > index && (max == Quantifier.INF || occurrences < max)) { + occurrences++ + index = nextIndex + nextIndex = innerSet.matches(index, testString, matchResult) + } + + // TODO: Do we need this check? + if (/* nextIndex < 0 && */ occurrences < quantifier.min) { + return -1 + } else { + return next.matches(index, testString, matchResult) + } + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt new file mode 100644 index 00000000000..726ee4f9fdb --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/PossessiveLeafQuantifierSet.kt @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Possessive quantifier node over a leaf node. + * - a{n,m}+; + * - a*+ == a{0, }+; + * - a?+ == a{0, 1}+; + * - a++ == a{1, }+; + * + * @author Nikolay A. Kuznetsov + */ +internal class PossessiveLeafQuantifierSet( + quant: Quantifier, + innerSet: LeafSet, + next: AbstractSet, type: Int +) : LeafQuantifierSet(quant, innerSet, next, type) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + var occurrences = 0 + + // TODO: Create a separate functions for these loops? + while (occurrences < min) { + if (index + leaf.charCount > testString.length) { + return -1 + } + + val shift = leaf.accepts(index, testString) + if (shift < 1) { + return -1 + } + index += shift + occurrences++ + } + + while ((max >= 0 || occurrences < max) && index + leaf.charCount <= testString.length) { + val shift = leaf.accepts(index, testString) + if (shift < 1) { + break + } + index += shift + occurrences++ + } + + return next.matches(index, testString, matchResult) + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/PreviousMatchSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/PreviousMatchSet.kt new file mode 100644 index 00000000000..03d4d8fd093 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/PreviousMatchSet.kt @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Node representing previous match (\G). + * + * @author Nikolay A. Kuznetsov + */ +internal class PreviousMatchSet : SimpleSet() { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + if (startIndex == matchResult.previousMatch) { + return next.matches(startIndex, testString, matchResult) + } + return -1 + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = false + override val name: String + get() = "PreviousMatchSet" +} + + + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/QuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/QuantifierSet.kt new file mode 100644 index 00000000000..480ae9d5b15 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/QuantifierSet.kt @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Base class for quantifiers. + * + * @author Nikolay A. Kuznetsov + */ +// TODO: check of innerSet must be nullable or not +internal abstract class QuantifierSet(open var innerSet: AbstractSet, override var next: AbstractSet, type: Int) + : SimpleSet(type) { + + // TODO: Looks like we need true by default here. + override fun first(set: AbstractSet): Boolean = + innerSet.first(set) || next.first(set) + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true + + override fun processSecondPassInternal(): AbstractSet { + val innerSet = this.innerSet + if (innerSet.secondPassVisited) { + this.innerSet = innerSet.processSecondPass() + } + + return super.processSecondPassInternal() + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/RangeSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/RangeSet.kt new file mode 100644 index 00000000000..6b83c5f2b1b --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/RangeSet.kt @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents node accepting single character from the given char class. (TODO: Not a surrogate character?) + * + * @author Nikolay A. Kuznetsov + */ +open internal class RangeSet(charClass: AbstractCharClass, val ignoreCase: Boolean = false) : LeafSet() { + + val chars: AbstractCharClass = charClass.instance + + override fun accepts(startIndex: Int, testString: CharSequence): Int { + if (ignoreCase) { + val char = testString[startIndex] + return if (chars.contains(char.toUpperCase()) || chars.contains(char.toLowerCase())) 1 else -1 + } else { + return if (chars.contains(testString[startIndex])) 1 else -1 + } + } + + override val name: String + get() = "range:" + (if (chars.alt) "^ " else " ") + chars.toString() + + override fun first(set: AbstractSet): Boolean { + return when (set) { + is CharSet -> AbstractCharClass.intersects(chars, set.char.toInt()) + is RangeSet -> AbstractCharClass.intersects(chars, set.chars) + is SupplementaryCharSet -> false + is SupplementaryRangeSet -> AbstractCharClass.intersects(chars, set.chars) + else -> true + } + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantGroupQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantGroupQuantifierSet.kt new file mode 100644 index 00000000000..ed3b0a92f92 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantGroupQuantifierSet.kt @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Reluctant version of the group quantifier set. + * + * @author Nikolay A. Kuznetsov + */ +internal class ReluctantGroupQuantifierSet( + quantifier: Quantifier, + innerSet: AbstractSet, + next: AbstractSet, + type: Int, + setCounter: Int +) : GroupQuantifierSet(quantifier, innerSet, next, type, setCounter) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + + if (!innerSet.hasConsumed(matchResult)) { + return next.matches(startIndex, testString, matchResult) + } + + // Fast case: '*' or {0, } - no need to count occurrences. + if (min == 0 && max == Quantifier.INF) { + val res = next.matches(startIndex, testString, matchResult) + return if (res < 0) { + innerSet.matches(startIndex, testString, matchResult) + } else { + res + } + } + + val enterCounter = matchResult.enterCounters[groupQuantifierIndex] + + // can't go inner set; + if (enterCounter >= max) { + matchResult.enterCounters[groupQuantifierIndex] = 0 // TODO: Do we need it? + return next.matches(startIndex, testString, matchResult) + } + + var nextIndex: Int + if (enterCounter >= min) { + nextIndex = next.matches(startIndex, testString, matchResult) + if (nextIndex < 0) { + matchResult.enterCounters[groupQuantifierIndex]++ + nextIndex = innerSet.matches(startIndex, testString, matchResult) + } else { + matchResult.enterCounters[groupQuantifierIndex] = 0 + return nextIndex + } + } else { + matchResult.enterCounters[groupQuantifierIndex]++ + nextIndex = innerSet.matches(startIndex, testString, matchResult) + } + + return nextIndex + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt new file mode 100644 index 00000000000..3702ab8f375 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/ReluctantLeafQuantifierSet.kt @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Reluctant quantifier node over a leaf node. + * - a{n,m}?; + * - a*? == a{0, }?; + * - a?? == a{0, 1}?; + * - a+? == a{1, }?; + * + * @author Nikolay A. Kuznetsov + */ +internal class ReluctantLeafQuantifierSet( + quant: Quantifier, + innerSet: LeafSet, + next: AbstractSet, + type: Int +) : LeafQuantifierSet(quant, innerSet, next, type) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + var occurrences = 0 + + while (occurrences < min) { + if (index + leaf.charCount > testString.length) { + return -1 + } + + val shift = leaf.accepts(index, testString) + if (shift < 1) { + return -1 + } + index += shift + occurrences++ + } + + // TODO: May be refactor this loop. + do { + var shift = next.matches(index, testString, matchResult) + if (shift >= 0) { + return shift + } + + if (index + leaf.charCount <= testString.length) { + shift = leaf.accepts(index, testString) + index += shift + occurrences++ + } + + } while (shift >= 1 && occurrences <= max) + + return -1 + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/SOLSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/SOLSet.kt new file mode 100644 index 00000000000..2fe71fc7988 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/SOLSet.kt @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * A node representing a '^' sign. + * Note: In Kotlin we use only the "anchoring bounds" mode when "^" matches beginning of a match region. + * See: http://docs.oracle.com/javase/8/docs/api/java/util/regex/Matcher.html#useAnchoringBounds-boolean- + * + * @author Nikolay A. Kuznetsov + */ +internal class SOLSet(val lt: AbstractLineTerminator, val multiline: Boolean = false) : SimpleSet() { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + if (!multiline) { + if (startIndex == 0) { + return next.matches(startIndex, testString, matchResult) + } + } else { + // TODO: In Kotlin JVM the empty string doesn't match to "^.$" pattern in multiline mode and matches in single-line mode. + // TODO: So do we need to implement this behaviour or we can match to the pattern in the both cases? + if (startIndex != testString.length + && (startIndex == 0 + || lt.isAfterLineTerminator(testString[startIndex - 1], testString[startIndex]))) { + return next.matches(startIndex, testString, matchResult) + } + } + return -1 + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = false + override val name: String + get() = "^" +} \ No newline at end of file diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/SequenceSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/SequenceSet.kt new file mode 100644 index 00000000000..09f06f09253 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/SequenceSet.kt @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * This class represents nodes constructed with character sequences. For + * example, lets consider regular expression: ".*word.*". During regular + * expression compilation phase character sequence w-o-r-d, will be represented + * with single node for the entire word. + * + * @author Nikolay A. Kuznetsov + */ +open internal class SequenceSet(substring: CharSequence, val ignoreCase: Boolean = false) : LeafSet() { + + /** Represents a character sequence used for matching/searching. */ + protected val patternString: String = substring.toString() + + override val name: String= "sequence: " + patternString + + override val charCount = substring.length + + // Overrides ======================================================================================================= + + /** Returns true if [index] points to a low surrogate following a high surrogate */ + private fun isLowSurrogateOfSupplement(string: CharSequence, index: Int): Boolean = + index < string.length && string[index].isLowSurrogate() && index > 0 && string[index - 1].isHighSurrogate() + + override fun accepts(startIndex: Int, testString: CharSequence): Int { + return if (testString.startsWith(patternString, startIndex, ignoreCase) + && !isLowSurrogateOfSupplement(testString, startIndex) + && !isLowSurrogateOfSupplement(testString, startIndex + patternString.length)) { + charCount + } else { + -1 + } + } + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + while (index < testString.length) { + index = testString.indexOf(patternString, index, ignoreCase) + if (index < 0) { + return -1 + } + // Check if we have a supplementary code point at the beginning or at the end of the string. + if (!isLowSurrogateOfSupplement(testString, index) + && !isLowSurrogateOfSupplement(testString, index + patternString.length) + && next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index++ + } + return -1 + } + + override fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = rightLimit + while (index >= leftLimit) { + index = testString.lastIndexOf(patternString, index, ignoreCase) + if (index < 0) { + return -1 + } + // Check if we have a supplementary code point at the beginning or at the end of the string. + if (!isLowSurrogateOfSupplement(testString, index) + && !isLowSurrogateOfSupplement(testString, index + patternString.length) + && next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index-- + } + return -1 + } + + // TODO: reread/rewrite + override fun first(set: AbstractSet): Boolean { + if (ignoreCase) { + return super.first(set) // TODO: Add correct checks for this case. + } + return when (set) { + is CharSet -> set.char == patternString[0] + is RangeSet -> set.accepts(0, patternString.substring(0, 1)) > 0 + // TODO: perfrom a char to codepoint converter. + is SupplementaryRangeSet -> set.contains(patternString[0]) || patternString.length > 1 && set.contains(Char.toCodePoint(patternString[0], patternString[1])) + is SupplementaryCharSet -> if (patternString.length > 1) set.codePoint == Char.toCodePoint(patternString[0], patternString[1]) + else false + else -> true + } + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/SingleSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/SingleSet.kt new file mode 100644 index 00000000000..ffbef8eefdf --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/SingleSet.kt @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Group node over subexpression without alternations. + * @author Nikolay A. Kuznetsov + */ +open internal class SingleSet(var kid: AbstractSet, fSet: FSet) : JointSet(listOf(), fSet) { + + var backReferencedSet: BackReferencedSingleSet? = null + + // Overrides (API) ================================================================================================= + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val start = matchResult.getStart(groupIndex) + matchResult.setStart(groupIndex, startIndex) + val shift = kid.matches(startIndex, testString, matchResult) + if (shift >= 0) { + return shift + } + matchResult.setStart(groupIndex, start) + return -1 + } + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val res = kid.find(startIndex, testString, matchResult) + if (res >= 0) + matchResult.setStart(groupIndex, res) + return res + } + + override fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val res = kid.findBack(leftLimit, rightLimit, testString, matchResult) + if (res >= 0) + matchResult.setStart(groupIndex, res) + return res + } + + override fun first(set: AbstractSet): Boolean = kid.first(set) + + // Second pass processing ========================================================================================== + + override fun processBackRefReplacement(): JointSet? { + /** + * We will store a reference to created BackReferencedSingleSet + * in [backReferencedSet] field. This is needed to process replacement + * of sets correctly since sometimes we cannot renew all references to + * detachable set in the current point of traverse. See + * QuantifierSet and AbstractSet processSecondPass() methods for + * more details. + */ + val result = BackReferencedSingleSet(this) + backReferencedSet = result + return result + } + + override fun processSecondPassInternal(): AbstractSet { + val fSet = this.fSet + if (fSet != null) { + this.fSet = fSet.processSecondPass() + } + kid = kid.processSecondPass() + return processBackRefReplacement() ?: this + } + + /** + * This method is used for traversing nodes after the first stage of compilation. + */ + override fun processSecondPass(): AbstractSet { + if (secondPassVisited) { + val fSet = this.fSet + if (fSet is FSet && fSet.isBackReferenced) { + assert(backReferencedSet != null) // secondPassVisited + return backReferencedSet!! + } + } + secondPassVisited = true + return processSecondPassInternal() + } + + // Backreferenced version of the class ============================================================================= + + /** + * Group node over subexpression without alternations. + * This node is used if current group is referenced via a backreference. + */ + internal class BackReferencedSingleSet(node: SingleSet) : SingleSet(node.kid, node.fSet as FSet) { + + /* + * This class is needed only for overwriting find() and findBack() methods of SingleSet class, which is being + * back referenced. The following example explains the need for such substitution: + * + * Let's consider the pattern ".*(.)\\1". + * Leading .* works as follows: finds line terminator and runs findBack from that point. + * `findBack` method in its turn (in contrast to matches) sets group boundaries on the back trace. + * Thus at the point we try to match back reference(\\1) groups are not yet set. + * + * To fix this problem we replace backreferenced groups with instances of this class, + * which will use matches instead of find; this will affect performance, but ensure correctness of the match. + */ + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + for (index in startIndex..testString.length) { + val oldStart = matchResult.getStart(groupIndex) + matchResult.setStart(groupIndex, index) + + val res = kid.matches(index, testString, matchResult) + if (res >= 0) { + return index + } else { + matchResult.setStart(groupIndex, oldStart) + } + } + return -1 + } + + override fun findBack(leftLimit: Int, rightLimit: Int, + testString: CharSequence, matchResult: MatchResultImpl): Int { + for (index in rightLimit downTo leftLimit) { + val oldStart = matchResult.getStart(groupIndex) + matchResult.setStart(groupIndex, index) + + val res = kid.matches(index, testString, matchResult) + if (res >= 0) { + return index + } else { + matchResult.setStart(groupIndex, oldStart) + } + } + return -1 + } + + override fun processBackRefReplacement(): JointSet? = null + } +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/SupplementaryCharSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/SupplementaryCharSet.kt new file mode 100644 index 00000000000..40284be6041 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/SupplementaryCharSet.kt @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Portions, Copyright © 1991-2005 Unicode, Inc. The following applies to Unicode. + * + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright © 1991-2005 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. Permission is + * hereby granted, free of charge, to any person obtaining a copy of the + * Unicode data files and any associated documentation (the "Data Files") + * or Unicode software and any associated documentation (the "Software") + * to deal in the Data Files or Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * and/or sell copies of the Data Files or Software, and to permit persons + * to whom the Data Files or Software are furnished to do so, provided that + * (a) the above copyright notice(s) and this permission notice appear with + * all copies of the Data Files or Software, (b) both the above copyright + * notice(s) and this permission notice appear in associated documentation, + * and (c) there is clear notice in each modified Data File or in the Software + * as well as in the documentation associated with the Data File(s) or Software + * that the data or software has been modified. + + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT + * OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + * + * 2. Additional terms from the Database: + * + * Copyright © 1995-1999 Unicode, Inc. All Rights reserved. + * + * Disclaimer + * + * The Unicode Character Database is provided as is by Unicode, Inc. + * No claims are made as to fitness for any particular purpose. No warranties + * of any kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been purchased + * on magnetic or optical media from Unicode, Inc., the sole remedy for any claim + * will be exchange of defective media within 90 days of receipt. This disclaimer + * is applicable for all other data files accompanying the Unicode Character Database, + * some of which have been compiled by the Unicode Consortium, and some of which + * have been supplied by other sources. + * + * Limitations on Rights to Redistribute This Data + * + * Recipient is granted the right to make copies in any form for internal + * distribution and to freely use the information supplied in the creation of + * products supporting the UnicodeTM Standard. The files in + * the Unicode Character Database can be redistributed to third parties or other + * organizations (whether for profit or not) as long as this notice and the disclaimer + * notice are retained. Information can be extracted from these files and used + * in documentation or programs, as long as there is an accompanying notice + * indicating the source. + */ + +package kotlin.text + +/** + * Represents node accepting single supplementary codepoint. + * + * @author Nikolay A. Kuznetsov + */ +//assumes int value of this supplementary codepoint as a parameter +// TODO: Tests crash here. Investigate why. +internal class SupplementaryCharSet(val codePoint: Int, ignoreCase: Boolean) + : SequenceSet(fromCharArray(Char.toChars(codePoint), 0, 2), ignoreCase) { + + override val name: String + get() = patternString + + override fun first(set: AbstractSet): Boolean { + // TODO: replace with when + return when (set) { + is SupplementaryCharSet -> set.codePoint == codePoint + is SupplementaryRangeSet -> set.contains(codePoint) + is CharSet, + is RangeSet -> false + else -> true + } + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/SupplementaryRangeSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/SupplementaryRangeSet.kt new file mode 100644 index 00000000000..edc4212bfc6 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/SupplementaryRangeSet.kt @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Portions, Copyright © 1991-2005 Unicode, Inc. The following applies to Unicode. + * + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright © 1991-2005 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. Permission is + * hereby granted, free of charge, to any person obtaining a copy of the + * Unicode data files and any associated documentation (the "Data Files") + * or Unicode software and any associated documentation (the "Software") + * to deal in the Data Files or Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * and/or sell copies of the Data Files or Software, and to permit persons + * to whom the Data Files or Software are furnished to do so, provided that + * (a) the above copyright notice(s) and this permission notice appear with + * all copies of the Data Files or Software, (b) both the above copyright + * notice(s) and this permission notice appear in associated documentation, + * and (c) there is clear notice in each modified Data File or in the Software + * as well as in the documentation associated with the Data File(s) or Software + * that the data or software has been modified. + + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT + * OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + * + * 2. Additional terms from the Database: + * + * Copyright © 1995-1999 Unicode, Inc. All Rights reserved. + * + * Disclaimer + * + * The Unicode Character Database is provided as is by Unicode, Inc. + * No claims are made as to fitness for any particular purpose. No warranties + * of any kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been purchased + * on magnetic or optical media from Unicode, Inc., the sole remedy for any claim + * will be exchange of defective media within 90 days of receipt. This disclaimer + * is applicable for all other data files accompanying the Unicode Character Database, + * some of which have been compiled by the Unicode Consortium, and some of which + * have been supplied by other sources. + * + * Limitations on Rights to Redistribute This Data + * + * Recipient is granted the right to make copies in any form for internal + * distribution and to freely use the information supplied in the creation of + * products supporting the UnicodeTM Standard. The files in + * the Unicode Character Database can be redistributed to third parties or other + * organizations (whether for profit or not) as long as this notice and the disclaimer + * notice are retained. Information can be extracted from these files and used + * in documentation or programs, as long as there is an accompanying notice + * indicating the source. + */ + +package kotlin.text + +/** + * Represents node accepting single character from the given char class. + * This character can be supplementary (2 chars needed to represent) or from + * basic multilingual pane (1 needed char to represent it). + * + * @author Nikolay A. Kuznetsov + */ +open internal class SupplementaryRangeSet(charClass: AbstractCharClass, val ignoreCase: Boolean = false): SimpleSet() { + + // TODO: may be add ignoreCase flag to CharClass + val chars = charClass.instance + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val rightBound = testString.length + if (startIndex >= rightBound) { + return -1 + } + + var index = startIndex + + val high = testString[index++] + val result = next.matches(index, testString, matchResult) + if (contains(high) && result >= 0) { + return result + } + + if (index < rightBound) { + val low = testString[index++] + if (Char.isSurrogatePair(high, low) && contains(Char.toCodePoint(high, low))) { + return next.matches(index, testString, matchResult) + } + } + + return -1 + } + + fun contains(char: Char): Boolean { + if (ignoreCase) { + return chars.contains(char.toUpperCase()) || chars.contains(char.toLowerCase()) + } else { + return chars.contains(char) + } + } + + fun contains(char: Int): Boolean { + return chars.contains(char) + } + + override val name: String + get() = "range:" + (if (chars.alt) "^ " else " ") + chars.toString() + + + override fun first(set: AbstractSet): Boolean { + return when(set) { + is SupplementaryCharSet -> AbstractCharClass.intersects(chars, set.codePoint) + is CharSet -> AbstractCharClass.intersects(chars, set.char.toInt()) + is SupplementaryRangeSet -> AbstractCharClass.intersects(chars, set.chars) + is RangeSet -> AbstractCharClass.intersects(chars, set.chars) + else -> true + } + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true +} + + + + + + + + + + + + + + + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/SurrogateCharSets.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/SurrogateCharSets.kt new file mode 100644 index 00000000000..cbdd5966de8 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/SurrogateCharSets.kt @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Portions, Copyright © 1991-2005 Unicode, Inc. The following applies to Unicode. + * + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright © 1991-2005 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. Permission is + * hereby granted, free of charge, to any person obtaining a copy of the + * Unicode data files and any associated documentation (the "Data Files") + * or Unicode software and any associated documentation (the "Software") + * to deal in the Data Files or Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * and/or sell copies of the Data Files or Software, and to permit persons + * to whom the Data Files or Software are furnished to do so, provided that + * (a) the above copyright notice(s) and this permission notice appear with + * all copies of the Data Files or Software, (b) both the above copyright + * notice(s) and this permission notice appear in associated documentation, + * and (c) there is clear notice in each modified Data File or in the Software + * as well as in the documentation associated with the Data File(s) or Software + * that the data or software has been modified. + + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT + * OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + * + * 2. Additional terms from the Database: + * + * Copyright © 1995-1999 Unicode, Inc. All Rights reserved. + * + * Disclaimer + * + * The Unicode Character Database is provided as is by Unicode, Inc. + * No claims are made as to fitness for any particular purpose. No warranties + * of any kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been purchased + * on magnetic or optical media from Unicode, Inc., the sole remedy for any claim + * will be exchange of defective media within 90 days of receipt. This disclaimer + * is applicable for all other data files accompanying the Unicode Character Database, + * some of which have been compiled by the Unicode Consortium, and some of which + * have been supplied by other sources. + * + * Limitations on Rights to Redistribute This Data + * + * Recipient is granted the right to make copies in any form for internal + * distribution and to freely use the information supplied in the creation of + * products supporting the UnicodeTM Standard. The files in + * the Unicode Character Database can be redistributed to third parties or other + * organizations (whether for profit or not) as long as this notice and the disclaimer + * notice are retained. Information can be extracted from these files and used + * in documentation or programs, as long as there is an accompanying notice + * indicating the source. + */ + +package kotlin.text + +/** + * This class represents low surrogate character. + * + * Note that we can use high and low surrogate characters + * that don't combine into supplementary code point. + * See http://www.unicode.org/reports/tr18/#Supplementary_Characters + * + * @author Nikolay A. Kuznetsov + */ +// TODO: Could we use just CharSet instead of this one? +internal class LowSurrogateCharSet(low: Char) : CharSet(low) { + + // TODO: Remove bounds from MatchResultImpl. + override fun accepts(startIndex: Int, testString: CharSequence): Int { + val result = super.accepts(startIndex, testString) + if (result < 0 || testString.isHighSurrogate(startIndex - 1)) { + return -1 + } + return result + } + + private fun CharSequence.isHighSurrogate(index: Int, leftBound: Int = 0, rightBound: Int = length) + = (index in leftBound until rightBound && this[index].isHighSurrogate()) + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + while (index < testString.length) { + index = testString.indexOf(char, index, ignoreCase) + if (index < 0) { + return -1 + } + if (!testString.isHighSurrogate(index - 1) + && next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index++ + } + return -1 + } + + override fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = rightLimit + while (index >= leftLimit) { + index = testString.lastIndexOf(char, index, ignoreCase) + if (index < 0) { + return -1 + } + if (!testString.isHighSurrogate(index - 1, leftLimit, rightLimit) + && next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index-- + } + return -1 + } + + override fun first(set: AbstractSet): Boolean { + return when(set) { + is LowSurrogateCharSet -> set.char == this.char + is CharSet, + is RangeSet, + is SupplementaryCharSet, + is SupplementaryRangeSet -> false + else -> true + } + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true +} + +/** + * This class represents high surrogate character. + */ +internal class HighSurrogateCharSet(high: Char) : CharSet(high) { + + override fun accepts(startIndex: Int, testString: CharSequence): Int { + val result = super.accepts(startIndex, testString) + if (result < 0 || testString.isLowSurrogate(startIndex + 1)) { + return -1 + } + return result + } + + private fun CharSequence.isLowSurrogate(index: Int, leftBound: Int = 0, rightBound: Int = length) + = (index in leftBound until rightBound && this[index].isLowSurrogate()) + + // TODO: We have a similar code here, in LowSurrogateCharSet and in CharSet. Reuse it somehow. + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + while (index < testString.length) { + index = testString.indexOf(char, index, ignoreCase) + if (index < 0) { + return -1 + } + // Remove params. + if (!testString.isLowSurrogate(index + 1) + && next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index++ + } + return -1 + } + + override fun findBack(leftLimit: Int, rightLimit: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = rightLimit + while (index >= leftLimit) { + index = testString.lastIndexOf(char, index, ignoreCase) + if (index < 0) { + return -1 + } + if (!testString.isLowSurrogate(index + 1, leftLimit, rightLimit) + && next.matches(index + charCount, testString, matchResult) >= 0) { + return index + } + index-- + } + return -1 + } + + override fun first(set: AbstractSet): Boolean { + return when (set) { + is HighSurrogateCharSet -> set.char == this.char + is CharSet, + is RangeSet, + is SupplementaryCharSet, + is SupplementaryRangeSet -> false + else -> true + } + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true +} + diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/SurrogateRangeSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/SurrogateRangeSet.kt new file mode 100644 index 00000000000..e14f1052127 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/SurrogateRangeSet.kt @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * Portions, Copyright © 1991-2005 Unicode, Inc. The following applies to Unicode. + * + * COPYRIGHT AND PERMISSION NOTICE + * + * Copyright © 1991-2005 Unicode, Inc. All rights reserved. Distributed under + * the Terms of Use in http://www.unicode.org/copyright.html. Permission is + * hereby granted, free of charge, to any person obtaining a copy of the + * Unicode data files and any associated documentation (the "Data Files") + * or Unicode software and any associated documentation (the "Software") + * to deal in the Data Files or Software without restriction, including without + * limitation the rights to use, copy, modify, merge, publish, distribute, + * and/or sell copies of the Data Files or Software, and to permit persons + * to whom the Data Files or Software are furnished to do so, provided that + * (a) the above copyright notice(s) and this permission notice appear with + * all copies of the Data Files or Software, (b) both the above copyright + * notice(s) and this permission notice appear in associated documentation, + * and (c) there is clear notice in each modified Data File or in the Software + * as well as in the documentation associated with the Data File(s) or Software + * that the data or software has been modified. + + * THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + * KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT + * OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS + * INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT + * OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS + * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + * PERFORMANCE OF THE DATA FILES OR SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall + * not be used in advertising or otherwise to promote the sale, use or other + * dealings in these Data Files or Software without prior written + * authorization of the copyright holder. + * + * 2. Additional terms from the Database: + * + * Copyright © 1995-1999 Unicode, Inc. All Rights reserved. + * + * Disclaimer + * + * The Unicode Character Database is provided as is by Unicode, Inc. + * No claims are made as to fitness for any particular purpose. No warranties + * of any kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been purchased + * on magnetic or optical media from Unicode, Inc., the sole remedy for any claim + * will be exchange of defective media within 90 days of receipt. This disclaimer + * is applicable for all other data files accompanying the Unicode Character Database, + * some of which have been compiled by the Unicode Consortium, and some of which + * have been supplied by other sources. + * + * Limitations on Rights to Redistribute This Data + * + * Recipient is granted the right to make copies in any form for internal + * distribution and to freely use the information supplied in the creation of + * products supporting the UnicodeTM Standard. The files in + * the Unicode Character Database can be redistributed to third parties or other + * organizations (whether for profit or not) as long as this notice and the disclaimer + * notice are retained. Information can be extracted from these files and used + * in documentation or programs, as long as there is an accompanying notice + * indicating the source. + */ + +package kotlin.text + +/* + * This class is a range that contains only surrogate characters. + * + * @author Nikolay A. Kuznetsov + */ +internal class SurrogateRangeSet(surrChars: AbstractCharClass) : RangeSet(surrChars) { + + override fun accepts(startIndex: Int, testString: CharSequence): Int { + val result = super.accepts(startIndex, testString) + when { + result < 0 || + testString.isHighSurrogate(startIndex - 1) && testString.isLowSurrogate(startIndex) || + testString.isHighSurrogate(startIndex) && testString.isLowSurrogate(startIndex + 1) -> + return -1 + } + return result + } + + private fun CharSequence.isHighSurrogate(index: Int, leftBound: Int = 0, rightBound: Int = length) + = (index in leftBound until rightBound && this[index].isHighSurrogate()) + + private fun CharSequence.isLowSurrogate(index: Int, leftBound: Int = 0, rightBound: Int = length) + = (index in leftBound until rightBound && this[index].isLowSurrogate()) + + override fun first(set: AbstractSet): Boolean { + return when (set) { + is SurrogateRangeSet -> true // TODO: May be add some check here. + is CharSet, + is RangeSet, + is SupplementaryCharSet, + is SupplementaryRangeSet -> false + else -> true + } + } + + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = true +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/UnifiedQuantifierSet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/UnifiedQuantifierSet.kt new file mode 100644 index 00000000000..7f34b068614 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/UnifiedQuantifierSet.kt @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Optimized greedy quantifier node ('*') for the case where there is no intersection with + * next node and normal quantifiers could be treated as greedy and possessive. + * + * @author Nikolay A. Kuznetsov + */ +// TODO: May be rename? +internal class UnifiedQuantifierSet(quant: LeafQuantifierSet) : LeafQuantifierSet(Quantifier.starQuantifier, quant.leaf, quant.next, quant.type) { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var index = startIndex + while (index + leaf.charCount <= testString.length && leaf.accepts(index, testString) > 0) { + index += leaf.charCount + } + + return next.matches(index, testString, matchResult) + } + + override fun find(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + var startSearch = next.find(startIndex, testString, matchResult) + if (startSearch < 0) + return -1 + + var result = startSearch + var index = startSearch - leaf.charCount + while (index >= startIndex && leaf.accepts(index, testString) > 0) { + result = index + index -= leaf.charCount + } + return result + } + + init { + innerSet.next = this // TODO: O_o + } +} diff --git a/runtime/src/main/kotlin/kotlin/text/regex/sets/WordBoundarySet.kt b/runtime/src/main/kotlin/kotlin/text/regex/sets/WordBoundarySet.kt new file mode 100644 index 00000000000..81ea93f2e40 --- /dev/null +++ b/runtime/src/main/kotlin/kotlin/text/regex/sets/WordBoundarySet.kt @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package kotlin.text + +/** + * Represents word boundary, checks current character and previous one. If they have different types returns true; + * + * @author Nikolay A. Kuznetsov + */ +internal class WordBoundarySet(var positive: Boolean) : SimpleSet() { + + override fun matches(startIndex: Int, testString: CharSequence, matchResult: MatchResultImpl): Int { + val curChar = if (startIndex >= testString.length) ' ' else testString[startIndex] + val prevChar = if (startIndex == 0) ' ' else testString[startIndex - 1] + + val right = curChar == ' ' || isSpace(curChar, startIndex, testString) + val left = prevChar == ' ' || isSpace(prevChar, startIndex - 1, testString) + + return if (left xor right xor positive) + -1 + else + next.matches(startIndex, testString, matchResult) + } + + /** Returns false, because word boundary does not consumes any characters and do not move string index. */ + override fun hasConsumed(matchResult: MatchResultImpl): Boolean = false + override val name: String + get() = "WordBoundarySet" + + private fun isSpace(char: Char, startIndex: Int, testString: CharSequence): Boolean { + if (char.isLetterOrDigit() || char == '_') { + return false + } + if (char.category == CharCategory.NON_SPACING_MARK) { + var index = startIndex + while (--index >= 0) { + val ch = testString[index] + when { + ch.isLetterOrDigit() -> return false + char.category != CharCategory.NON_SPACING_MARK -> return true + } + } + } + return true + } +} + + +