Commonize CharCategory and related functions #KT-39177 #KT-43216 #KT-39906 #KT-30652
This commit is contained in:
Abduqodiri Qurbonzoda
@@ -0,0 +1,84 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package kotlin.text
|
||||
|
||||
//
|
||||
// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt
|
||||
// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib
|
||||
//
|
||||
|
||||
// 1343 ranges totally
|
||||
private object Category {
|
||||
val decodedRangeStart: IntArray
|
||||
val decodedRangeCategory: IntArray
|
||||
|
||||
init {
|
||||
val toBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
|
||||
val fromBase64 = IntArray(128)
|
||||
for (i in toBase64.indices) {
|
||||
fromBase64[toBase64[i].toInt()] = i
|
||||
}
|
||||
|
||||
// rangeStartDiff.length = 1482
|
||||
val rangeStartDiff = "gBCFEDCKCDCaDDaDBhBCEEDDDDDEDXBHYBH5BRwBGDCHDCIDFHDCHFDCDEIRTEE7BGHDDJlCBbSEMOFGERwDEDDDDECEFCRBJhBFDCYFFCCzBvBjBBFC3BOhDBmBDGpBDDCtBBJIbEECLGDFCLDCgBBKVKEDiDDHCFECECKCEODBebC5CLBOKhBJDDDDWEBHFCFCPBZDEL1BVBSLPBgBB2BDBDICFBHKCCKCPDBHEDWBHEDDDDEDEDIBDGDCKCCGDDDCGECCWBFMDDCDEDDCHDDHKDDBKDBHFCWBFGFDBDDFEDBPDDKCHBGDCHEDWBFGFDCEDEDBHDDGDCKCGJEGDBFDDFDDDDDMEFDBFDCGBOKDFDFDCGFCXBQDDDDDBEGEDFDDKHBHDDGFCXBKBFCEFCFCHCHECCKDNCCHFCoBEDECFDDDDHDCCKJBGDCSDYBJEHBFDDEBIGKDCMuBFHEBGBIBKCkBFBFBXEIFJDFDGCKCEgBBDPEDGKKGECIBkBEOBDFFLBkBBIBEFFEClBrBCEBEGDBKGGDDDDDCHDENDCFEKDDlBDDFrBCDpKBECGEECpBBEChBBECGEECPB5BBECjCCDJUDQKG2CCGDsTCRBaCDrCDDIHNBEDLSDCJSCMLFCCM0BDHGFLBFDDKGKGEFDDBKGjBB1BHFChBDFmCKfDDDDDDCGDCFDKeCFLsBEaGKBDiBXDDD1BDGDEIGJEKGKGHBGCMF/BEBvBCEDDFHEKHKJJDDeDDGDKsBFEDCIEkBIICCDFKDDKeGCJHrBCDIIDBNBHEBEFDBFsB/BNBiBlB6BBF1EIiDJIGCGCIIIIGCGCIIIIOCIIIIIIDFEDDBFEDDDDEBDIFDDFEDBLFGCEEICFBJCDEDCLDKBFBKCCGDDKDDNDgBQNEBDMPFFDEDEBFFHECEBEEDFBEDDQjBCEDEFFCCJHBeEEfsIIEUCHCxCBeZoBGlCZLV8BuCW3FBJB2BIvDB4HOesBFCfKQgIjEW/BEgBCiIwBVCGnBCgBBpDvBBuBEDBHEFGCCjDCGEDCFCFlBDDF4BHCOBXJHBHBHBHBHBHBHBHBgBCECGHGEDIFBKCEDMEtBaB5CM2GaMEDDCKCGFCJEDFDDDC2CDDDB6CDCFrBB+CDEKgBkBMQfBKeIBPgBKnBPgKguGgC9vUDVB3jBD3BJoBGCsIBDQKCUuBDDKCcCCmCKCGIXJCNC/BBHGKDECEVFBEMCEEBqBDDGDFDXDCEBDGEG0BEICyBQCICKGSGDEBKcICXLCLBdDDBvBDECCDNCKECFCJKFBpBFEDCJDBICCKCEQBGDDByBEDCEFBYDCLEDDCKGCGCGJHBHBrBBEJDEwCjBIDCKGk9KMXExBEggCgoGuLCqDmBHMFFCKBNBFBIsDQRrLCQgCC2BoBMCCQGEGQDCQDDDDFDGDECEEFBnEEBFEDCKCDCaDDaDBFCKBtBCfDGCGCFEDDDDCECKDC"
|
||||
val diff = decodeVarLenBase64(rangeStartDiff, fromBase64, 1342)
|
||||
val start = IntArray(diff.size + 1)
|
||||
for (i in diff.indices) {
|
||||
start[i + 1] = start[i] + diff[i]
|
||||
}
|
||||
decodedRangeStart = start
|
||||
|
||||
// rangeCategory.length = 2033
|
||||
val rangeCategory = "PsY44a41W54UYJYZYB14W7XC15WZPsYa84bl9Zw8b85Lr7C44brlerrYBZBCZCiBiBiBhCiiBhChiBhiCBhhChiCihBhChCChiBhChiClBCFhjCiBiBihDhiBhCCihBiBBhCCFCEbEbEb7EbGhCk7BixRkiCi4BRbh4BhRhCBRBCiiBBCiBChiZBCBCiBcGHhChCiBRBxxEYC40Rx8c6RGUm4GRFRFYRQZ44acG4wRYFEFGJYllGFlYGwcGmkEmcGFJFl8cYxwFGFGRFGFRJFGkkcYkxRm6aFGEGmmEmEGRYRFGxxYFRFRFRGQGIFmIFIGIooGFGFGYJ4EFmoIRFlxRlxRFRFxlRxlFllRxmFIGxxIoxRomFRIRxlFlmGRJFaL86F4mRxmGoRFRFRFRFllRxGIGRxmGxmGmxRxGRFlRRJmmFllGYRmmIRFllRlRFRFllRFxxGFIGmmRoxImxRFRllGmxRJ4aRFGxmIoRFlxRlxRFRFllRFxxGlImoGmmRxoIxoIGRmmIRxlFlmGRJ8FLRxmFFRFllRllRxxFlRlxRxlFRFRFRooGRIooRomRxFRIRJLc8aRmoIoGFllRlRFRFRlmGmoIooRGRGRxmGFRllGmxRJRYL8lGooYFllRlRFRFRFRmlIIxGooRGRIRlxFGRJxlFRGIFllRlRFlmGIGxIooRomF8xRxxFllILFGRJLcFxmIoRFRFRFxlRFRxxGxxIooGmmRRIRJxxIoYRFllGGRaFEGYJYRxlFRFRFlRFllGGlxRFxEGRJRFRFcY84c8mGcJL8G1WIFRFRGIGmmYFGRGRcGc88RYcYRFIGIGmmIomGFJYFooGmlFllGmmFIFIFGFmoIGIomFJIm8cBhRRxxBC4ECFRFRFlRFRFRFRFRFRFlRFRFRFRFRFRGYLRFcRBRCxxUF8YFMF1WRFYKFRFRFGRFGYRFGRFllRlRGRFmmIGIooGGY44E46FmxRJRLRY44U44GmmQRJRFEFRFGFlGRFRFxmGmoIooGmoIoxRxxIoGIGRxxcx4YJFRFRFRFRJLRcFmmIomRx4YFoGGmRomIGIGmxRJRJRYEYRGmmHRGIFmIGmIIooGFRJYcGcRmmIFomGmmIomGmlFJFmoGooGGIRYFIGIGRYJRFJFEYCRBRBYRGYGIGFGFllGomGFRCECECEGRGhCCiBCBCRBRCBCBCRBRCxBCBCRCDCDCDCiiRBj7CbCiiRBj7b7iCiiRxiCBRbCBbxxCiiRBj7bRMQUY9+V9+VYtOQMY9eY43X44Z1WY54XYMQRQrERLZ12ELZ12RERaRGHGHGR88B88BihBhiChhC8hcZBc8BB8CBCFi8cihBZBC8Z8CLKhCKr8cRZcZc88ZcZc85Z8ZcZc1WcZc1WcZcZcZcRcRLcLcZcZcZcZc1WLcZ1WZ1WZcZ1WZ1WZ1WZcZcZcRcRcBRCixBBCiBBihCCEBhCCchCGhCRY44LCiRRxxCFRkYRGFRFRFRFRFRFRFRFRFRGY9eY49eY44U49e49e1WYEYUY04VY48cRcRcRcRcRs4Y48ElK1Wc1W12U2cKGooUE88KqqEl4c8RFxxGm7bkkFUF4kEkFRFRFx8cLcFcRFcRLcLcLcLcLcFcFRFEFRcRFEYFEYFJFRhClmHnnYG4EhCEGFKGYRbEbhCCiBECiBhCk7bhClBihCiBBCBhCRhiBhhCCRhiFkkCFlGllGllGFooGmIcGRL88aRFYRIFIGRYJRGFYl4FGJFGYFGIRYFRGIFmoIGIGIYxEJRYFmEFJFRFGmoImoIGRFGFmIRJRYFEFcloGIFmlGmlFGFlmGFRllEYFomGo4YlkEoGRFRFRFRFRFRCbECk7bRCFooG4oGRJRFRFRFRTSFRFRCRCRlGFZFRFRlxFFbRF2VRFRFRF6cRGY41WRG40UX1W44V24Y44X33Y44R44U1WY50Z5R46YRFRFxxQY44a41W54UYJYZYB14W7XC15WZ12YYFEFEFRFRFRFlxRllRxxa65b86axcZcRQcR"
|
||||
decodedRangeCategory = decodeVarLenBase64(rangeCategory, fromBase64, 1343)
|
||||
}
|
||||
}
|
||||
|
||||
private fun categoryValueFrom(code: Int, ch: Int): Int {
|
||||
return when {
|
||||
code < 0x20 -> code
|
||||
code < 0x400 -> if ((ch and 1) == 1) code shr 5 else code and 0x1f
|
||||
else ->
|
||||
when (ch % 3) {
|
||||
2 -> code shr 10
|
||||
1 -> (code shr 5) and 0x1f
|
||||
else -> code and 0x1f
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the Unicode general category of this character as an Int.
|
||||
*/
|
||||
internal fun Char.getCategoryValue(): Int {
|
||||
val ch = this.toInt()
|
||||
|
||||
val index = binarySearchRange(Category.decodedRangeStart, ch)
|
||||
val start = Category.decodedRangeStart[index]
|
||||
val code = Category.decodedRangeCategory[index]
|
||||
val value = categoryValueFrom(code, ch - start)
|
||||
|
||||
return if (value == 17) CharCategory.UNASSIGNED.value else value
|
||||
}
|
||||
|
||||
internal fun decodeVarLenBase64(base64: String, fromBase64: IntArray, resultLength: Int): IntArray {
|
||||
val result = IntArray(resultLength)
|
||||
var index = 0
|
||||
var int = 0
|
||||
var shift = 0
|
||||
for (char in base64) {
|
||||
val sixBit = fromBase64[char.toInt()]
|
||||
int = int or ((sixBit and 0x1f) shl shift)
|
||||
if (sixBit < 0x20) {
|
||||
result[index++] = int
|
||||
int = 0
|
||||
shift = 0
|
||||
} else {
|
||||
shift += 5
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package kotlin.text
|
||||
|
||||
//
|
||||
// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt
|
||||
// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib
|
||||
//
|
||||
|
||||
// 37 ranges totally
|
||||
private object Digit {
|
||||
internal val rangeStart = intArrayOf(
|
||||
0x0030, 0x0660, 0x06f0, 0x07c0, 0x0966, 0x09e6, 0x0a66, 0x0ae6, 0x0b66, 0x0be6, 0x0c66, 0x0ce6, 0x0d66, 0x0de6, 0x0e50, 0x0ed0, 0x0f20, 0x1040, 0x1090, 0x17e0,
|
||||
0x1810, 0x1946, 0x19d0, 0x1a80, 0x1a90, 0x1b50, 0x1bb0, 0x1c40, 0x1c50, 0xa620, 0xa8d0, 0xa900, 0xa9d0, 0xa9f0, 0xaa50, 0xabf0, 0xff10,
|
||||
)
|
||||
}
|
||||
|
||||
internal fun binarySearchRange(array: IntArray, needle: Int): Int {
|
||||
var bottom = 0
|
||||
var top = array.size - 1
|
||||
var middle = -1
|
||||
var value = 0
|
||||
while (bottom <= top) {
|
||||
middle = (bottom + top) / 2
|
||||
value = array[middle]
|
||||
if (needle > value)
|
||||
bottom = middle + 1
|
||||
else if (needle == value)
|
||||
return middle
|
||||
else
|
||||
top = middle - 1
|
||||
}
|
||||
return middle - (if (needle < value) 1 else 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `true` if this character is a digit.
|
||||
*/
|
||||
internal fun Char.isDigitImpl(): Boolean {
|
||||
val ch = this.toInt()
|
||||
val index = binarySearchRange(Digit.rangeStart, ch)
|
||||
val high = Digit.rangeStart[index] + 9
|
||||
return ch <= high
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package kotlin.text
|
||||
|
||||
//
|
||||
// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt
|
||||
// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib
|
||||
//
|
||||
|
||||
// 222 ranges totally
|
||||
private object Letter {
|
||||
val decodedRangeStart: IntArray
|
||||
val decodedRangeLength: IntArray
|
||||
val decodedRangeCategory: IntArray
|
||||
|
||||
init {
|
||||
val toBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
|
||||
val fromBase64 = IntArray(128)
|
||||
for (i in toBase64.indices) {
|
||||
fromBase64[toBase64[i].toInt()] = i
|
||||
}
|
||||
|
||||
// rangeStartDiff.length = 356
|
||||
val rangeStartDiff = "hCgBpCQGYHZH5BRpBPPPPPPRMP5BPPlCPP6BkEPPPPcPXPzBvBrB3BOiDoBHwD+E3DauCnFmBmB2D6E1BlBTiBmBlBP5BhBiBrBvBjBqBnBPRtBiCmCtBlB0BmB5BiB7BmBgEmChBZgCoEoGVpBSfRhBPqKQ2BwBYoFgB4CJuTiEvBuCuDrF5DgEgFlJ1DgFmBQtBsBRGsB+BPiBlD1EIjDPRPPPQPPPPPGQSQS/DxENVNU+B9zCwBwBPPCkDPNnBPqDYY1R8B7FkFgTgwGgwUwmBgKwBuBScmEP/BPPPPPPrBP8B7F1B/ErBqC6B7BiBmBfQsBUwCw/KwqIwLwETPcPjQgJxFgBlBsD"
|
||||
val diff = decodeVarLenBase64(rangeStartDiff, fromBase64, 222)
|
||||
val start = IntArray(diff.size)
|
||||
for (i in diff.indices) {
|
||||
if (i == 0) start[i] = diff[i]
|
||||
else start[i] = start[i - 1] + diff[i]
|
||||
}
|
||||
decodedRangeStart = start
|
||||
|
||||
// rangeLength.length = 328
|
||||
val rangeLength = "aaMBXHYH5BRpBPPPPPPRMP5BPPlCPPzBDOOPPcPXPzBvBjB3BOhDmBBpB7DoDYxB+EiBP1DoExBkBQhBekBPmBgBhBctBiBMWOOXhCsBpBkBUV3Ba4BkB0DlCgBXgBtD4FSdBfPhBPpKP0BvBXjEQ2CGsT8DhBtCqDpFvD1D3E0IrD2EkBJrBDOBsB+BPiBlB1EIjDPPPPPPPPPPPGPPMNLsBNPNPKCvBvBPPCkDPBmBPhDXXgD4B6FzEgDguG9vUtkB9JcuBSckEP/BPPPPPPBPf4FrBjEhBpC3B5BKaWPrBOwCk/KsCuLqDHPbPxPsFtEaaqDL"
|
||||
decodedRangeLength = decodeVarLenBase64(rangeLength, fromBase64, 222)
|
||||
|
||||
// rangeCategory.length = 959
|
||||
val rangeCategory = "GFjgggUHGGFFZZZmzpz5qB6s6020B60ptltB6smt2sB60mz22B1+vv+8BZZ5s2850BW5q1ymtB506smzBF3q1q1qB1q1q1+Bgii4wDTm74g3KiggxqM60q1q1Bq1o1q1BF1qlrqrBZ2q5wprBGFZWWZGHFsjiooLowgmOowjkwCkgoiIk7ligGogiioBkwkiYkzj2oNoi+sbkwj04DghhkQ8wgiYkgoioDsgnkwC4gikQ//v+85BkwvoIsgoyI4yguI0whiwEowri4CoghsJowgqYowgm4DkwgsY/nwnzPowhmYkg6wI8yggZswikwHgxgmIoxgqYkwgk4DkxgmIkgoioBsgssoBgzgyI8g9gL8g9kI0wgwJoxgkoC0wgioFkw/wI0w53iF4gioYowjmgBHGq1qkgwBF1q1q8qBHwghuIwghyKk0goQkwgoQk3goQHGFHkyg0pBgxj6IoinkxDswno7Ikwhz9Bo0gioB8z48Rwli0xN0mpjoX8w78pDwltoqKHFGGwwgsIHFH3q1q16BFHWFZ1q10q1B2qlwq1B1q10q1B2q1yq1B6q1gq1Biq1qhxBir1qp1Bqt1q1qB1g1q1+B//3q16B///q1qBH/qlqq9Bholqq9B1i00a1q10qD1op1HkwmigEigiy6Cptogq1Bixo1kDq7/j00B2qgoBWGFm1lz50B6s5q1+BGWhggzhwBFFhgk4//Bo2jigE8wguI8wguI8wgugUog1qoB4qjmIwwi2KgkYHHH4lBgiFWkgIWoghssMmz5smrBZ3q1y50B5sm7gzBtz1smzB5smz50BqzqtmzB5sgzqzBF2/9//5BowgoIwmnkzPkwgk4C8ys65BkgoqI0wgy6FghquZo2giY0ghiIsgh24B4ghsQ8QF/v1q1OFs0O8iCHHF1qggz/B8wg6Iznv+//B08QgohsjK0QGFk7hsQ4gB"
|
||||
decodedRangeCategory = decodeVarLenBase64(rangeCategory, fromBase64, 222)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `true` if this character is a letter.
|
||||
*/
|
||||
internal fun Char.isLetterImpl(): Boolean {
|
||||
return getLetterType() != 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `true` if this character is a lower case letter.
|
||||
*/
|
||||
internal fun Char.isLowerCaseImpl(): Boolean {
|
||||
return getLetterType() == 1
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns `true` if this character is an upper case letter.
|
||||
*/
|
||||
internal fun Char.isUpperCaseImpl(): Boolean {
|
||||
return getLetterType() == 2
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns
|
||||
* - `1` if the character is a lower case letter,
|
||||
* - `2` if the character is an upper case letter,
|
||||
* - `3` if the character is a letter but not a lower or upper case letter,
|
||||
* - `0` otherwise.
|
||||
*/
|
||||
private fun Char.getLetterType(): Int {
|
||||
val ch = this.toInt()
|
||||
val index = binarySearchRange(Letter.decodedRangeStart, ch)
|
||||
|
||||
val rangeStart = Letter.decodedRangeStart[index]
|
||||
val rangeEnd = rangeStart + Letter.decodedRangeLength[index] - 1
|
||||
val code = Letter.decodedRangeCategory[index]
|
||||
|
||||
if (ch > rangeEnd) {
|
||||
return 0
|
||||
}
|
||||
|
||||
val lastTwoBits = code and 0x3
|
||||
|
||||
if (lastTwoBits == 0) { // gap pattern
|
||||
var shift = 2
|
||||
var threshold = rangeStart
|
||||
for (i in 0..1) {
|
||||
threshold += (code shr shift) and 0x7f
|
||||
if (threshold > ch) {
|
||||
return 3
|
||||
}
|
||||
shift += 7
|
||||
threshold += (code shr shift) and 0x7f
|
||||
if (threshold > ch) {
|
||||
return 0
|
||||
}
|
||||
shift += 7
|
||||
}
|
||||
return 3
|
||||
}
|
||||
|
||||
if (code <= 0x7) {
|
||||
return lastTwoBits
|
||||
}
|
||||
|
||||
val distance = (ch - rangeStart)
|
||||
val shift = if (code <= 0x1F) distance % 2 else distance
|
||||
return (code shr (2 * shift)) and 0x3
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
|
||||
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
|
||||
*/
|
||||
|
||||
package kotlin.text
|
||||
|
||||
//
|
||||
// NOTE: THIS FILE IS AUTO-GENERATED by the GenerateUnicodeData.kt
|
||||
// See: https://github.com/JetBrains/kotlin/tree/master/libraries/stdlib
|
||||
//
|
||||
|
||||
// 9 ranges totally
|
||||
/**
|
||||
* Returns `true` if this character is a whitespace.
|
||||
*/
|
||||
internal fun Char.isWhitespaceImpl(): Boolean {
|
||||
val ch = this.toInt()
|
||||
return ch in 0x0009..0x000d
|
||||
|| ch in 0x001c..0x0020
|
||||
|| ch == 0x00a0
|
||||
|| ch > 0x1000 && (
|
||||
ch == 0x1680
|
||||
|| ch in 0x2000..0x200a
|
||||
|| ch == 0x2028
|
||||
|| ch == 0x2029
|
||||
|| ch == 0x202f
|
||||
|| ch == 0x205f
|
||||
|| ch == 0x3000
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user