[Unicode] Add convenience APIs to Unicode encodings

Add convenience APIs to the stdlib's Unicode encodings:

* Unicode.UTF16
  * isASCII
  * isSurrogate
* Unicode.UTF8
  * isASCII
  * width
* Unicode.UTF32
  * isASCII
* Unicode.ASCII
  * isASCII

Tests added

Author: milseman

stdlib/public/core/ASCII.swift

@@ -23,6 +23,10 @@ extension Unicode.ASCII : Unicode.Encoding {
     return EncodedScalar(0x1a) // U+001A SUBSTITUTE; best we can do for ASCII
   }
 
+  /// Returns whether the given code unit represents an ASCII scalar
+  @_alwaysEmitIntoClient
+  public static func isASCII(_ x: CodeUnit) -> Bool { return UTF8.isASCII(x) }
+
   @inline(__always)
   @inlinable
   public static func _isScalar(_ x: CodeUnit) -> Bool {

stdlib/public/core/LegacyABI.swift

@@ -13,3 +13,23 @@
 // This file contains non-API (or underscored) declarations that are needed to
 // be kept around for ABI compatibility
 
+extension Unicode.UTF16 {
+  @available(*, unavailable, renamed: "Unicode.UTF16.isASCII")
+  @inlinable
+  public static func _isASCII(_ x: CodeUnit) -> Bool  {
+    return Unicode.UTF16.isASCII(x)
+  }
+}
+
+@available(*, unavailable, renamed: "Unicode.UTF8.isASCII")
+@inlinable
+internal func _isASCII(_ x: UInt8) -> Bool {
+  return Unicode.UTF8.isASCII(x)
+}
+
+@available(*, unavailable, renamed: "Unicode.UTF8.isContinuation")
+@inlinable
+internal func _isContinuation(_ x: UInt8) -> Bool {
+  return UTF8.isContinuation(x)
+}
+

stdlib/public/core/StringComparison.swift

@@ -239,7 +239,7 @@ private func _findBoundary(
   }
 
   // Back up to scalar boundary
-  while _isContinuation(utf8[_unchecked: idx]) {
+  while UTF8.isContinuation(utf8[_unchecked: idx]) {
     idx &-= 1
   }
 

stdlib/public/core/StringNormalization.swift

@@ -108,7 +108,7 @@ extension UnsafeBufferPointer where Element == UInt8 {
     if index == 0 || index == count {
       return true
     }
-    assert(!_isContinuation(self[_unchecked: index]))
+    assert(!UTF8.isContinuation(self[_unchecked: index]))
 
     // Sub-300 latiny fast-path
     if self[_unchecked: index] < 0xCC { return true }
@@ -165,7 +165,7 @@ extension UnsafeBufferPointer where Element == UInt8 {
       _internalInvariant(index == count)
       return true
     }
-    return !_isContinuation(self[index])
+    return !UTF8.isContinuation(self[index])
   }
 
 }

stdlib/public/core/StringUTF8Validation.swift

@@ -7,19 +7,15 @@ private func _isNotOverlong_F0(_ x: UInt8) -> Bool {
 }
 
 private func _isNotOverlong_F4(_ x: UInt8) -> Bool {
-  return _isContinuation(x) && x <= 0x8F
+  return UTF8.isContinuation(x) && x <= 0x8F
 }
 
 private func _isNotOverlong_E0(_ x: UInt8) -> Bool {
   return (0xA0...0xBF).contains(x)
 }
 
 private func _isNotOverlong_ED(_ x: UInt8) -> Bool {
-  return _isContinuation(x) && x <= 0x9F
-}
-
-private func _isASCII_cmp(_ x: UInt8) -> Bool {
-  return x <= 0x7F
+  return UTF8.isContinuation(x) && x <= 0x9F
 }
 
 internal struct UTF8ExtraInfo: Equatable {
@@ -48,7 +44,7 @@ internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationR
     guard f(cu) else { throw UTF8ValidationError() }
   }
   @inline(__always) func guaranteeContinuation() throws {
-    try guaranteeIn(_isContinuation)
+    try guaranteeIn(UTF8.isContinuation)
   }
 
   func _legacyInvalidLengthCalculation(_ _buffer: (_storage: UInt32, ())) -> Int {
@@ -94,7 +90,7 @@ internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationR
     var endIndex = buf.startIndex
     var iter = buf.makeIterator()
     _ = iter.next()
-    while let cu = iter.next(), !_isASCII(cu) && !_isUTF8MultiByteLeading(cu) {
+    while let cu = iter.next(), UTF8.isContinuation(cu) {
       endIndex += 1
     }
     let illegalRange = Range(buf.startIndex...endIndex)
@@ -107,7 +103,7 @@ internal func validateUTF8(_ buf: UnsafeBufferPointer<UInt8>) -> UTF8ValidationR
   do {
     var isASCII = true
     while let cu = iter.next() {
-      if _isASCII(cu) { lastValidIndex &+= 1; continue }
+      if UTF8.isASCII(cu) { lastValidIndex &+= 1; continue }
       isASCII = false
       if _slowPath(!_isUTF8MultiByteLeading(cu)) {
         throw UTF8ValidationError()

stdlib/public/core/StringUTF8View.swift

@@ -415,7 +415,7 @@ extension String.UTF8View {
 
     let (scalar, scalarLen) = _guts.foreignErrorCorrectedScalar(
       startingAt: i.strippingTranscoding)
-    let utf8Len = _numUTF8CodeUnits(scalar)
+    let utf8Len = UTF8.width(scalar)
 
     if utf8Len == 1 {
       _internalInvariant(i.transcodedOffset == 0)
@@ -442,7 +442,7 @@ extension String.UTF8View {
 
     let (scalar, scalarLen) = _guts.foreignErrorCorrectedScalar(
       endingAt: i)
-    let utf8Len = _numUTF8CodeUnits(scalar)
+    let utf8Len = UTF8.width(scalar)
     return i.encoded(offsetBy: -scalarLen).transcoded(withOffset: utf8Len &- 1)
   }
 

stdlib/public/core/StringUnicodeScalarView.swift

@@ -418,7 +418,7 @@ extension String.UnicodeScalarView {
   internal func _foreignIndex(after i: Index) -> Index {
     _internalInvariant(_guts.isForeign)
     let cu = _guts.foreignErrorCorrectedUTF16CodeUnit(at: i)
-    let len = _isLeadingSurrogate(cu) ? 2 : 1
+    let len = UTF16.isLeadSurrogate(cu) ? 2 : 1
 
     return i.encoded(offsetBy: len)
   }
@@ -429,7 +429,7 @@ extension String.UnicodeScalarView {
     _internalInvariant(_guts.isForeign)
     let priorIdx = i.priorEncoded
     let cu = _guts.foreignErrorCorrectedUTF16CodeUnit(at: priorIdx)
-    let len = _isTrailingSurrogate(cu) ? 2 : 1
+    let len = UTF16.isTrailSurrogate(cu) ? 2 : 1
 
     return i.encoded(offsetBy: -len)
   }