core/slice/
ascii.rs

1//! Operations on ASCII `[u8]`.
2
3use core::ascii::EscapeDefault;
4
5use crate::fmt::{self, Write};
6#[cfg(not(all(target_arch = "loongarch64", target_feature = "lsx")))]
7use crate::intrinsics::const_eval_select;
8use crate::{ascii, iter, ops};
9#[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
10use crate::{
11    iter::{Filter, FusedIterator},
12    slice::Split,
13    str::{BytesIsNotEmpty, IsAsciiWhitespace},
14};
15
16impl [u8] {
17    /// Checks if all bytes in this slice are within the ASCII range.
18    ///
19    /// An empty slice returns `true`.
20    #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
21    #[rustc_const_stable(feature = "const_slice_is_ascii", since = "1.74.0")]
22    #[must_use]
23    #[inline]
24    pub const fn is_ascii(&self) -> bool {
25        is_ascii(self)
26    }
27
28    /// If this slice [`is_ascii`](Self::is_ascii), returns it as a slice of
29    /// [ASCII characters](`ascii::Char`), otherwise returns `None`.
30    #[unstable(feature = "ascii_char", issue = "110998")]
31    #[must_use]
32    #[inline]
33    pub const fn as_ascii(&self) -> Option<&[ascii::Char]> {
34        if self.is_ascii() {
35            // SAFETY: Just checked that it's ASCII
36            Some(unsafe { self.as_ascii_unchecked() })
37        } else {
38            None
39        }
40    }
41
42    /// Converts this slice of bytes into a slice of ASCII characters,
43    /// without checking whether they're valid.
44    ///
45    /// # Safety
46    ///
47    /// Every byte in the slice must be in `0..=127`, or else this is UB.
48    #[unstable(feature = "ascii_char", issue = "110998")]
49    #[must_use]
50    #[inline]
51    pub const unsafe fn as_ascii_unchecked(&self) -> &[ascii::Char] {
52        let byte_ptr: *const [u8] = self;
53        let ascii_ptr = byte_ptr as *const [ascii::Char];
54        // SAFETY: The caller promised all the bytes are ASCII
55        unsafe { &*ascii_ptr }
56    }
57
58    /// Checks that two slices are an ASCII case-insensitive match.
59    ///
60    /// Same as `to_ascii_lowercase(a) == to_ascii_lowercase(b)`,
61    /// but without allocating and copying temporaries.
62    #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
63    #[rustc_const_stable(feature = "const_eq_ignore_ascii_case", since = "1.89.0")]
64    #[must_use]
65    #[inline]
66    pub const fn eq_ignore_ascii_case(&self, other: &[u8]) -> bool {
67        if self.len() != other.len() {
68            return false;
69        }
70
71        #[cfg(any(
72            all(target_arch = "x86_64", target_feature = "sse2"),
73            all(target_arch = "aarch64", target_feature = "neon")
74        ))]
75        {
76            const CHUNK_SIZE: usize = 16;
77            // The following function has two invariants:
78            // 1. The slice lengths must be equal, which we checked above.
79            // 2. The slice lengths must greater than or equal to N, which this
80            //    if-statement is checking.
81            if self.len() >= CHUNK_SIZE {
82                return self.eq_ignore_ascii_case_chunks::<CHUNK_SIZE>(other);
83            }
84        }
85
86        self.eq_ignore_ascii_case_simple(other)
87    }
88
89    /// ASCII case-insensitive equality check without chunk-at-a-time
90    /// optimization.
91    #[inline]
92    const fn eq_ignore_ascii_case_simple(&self, other: &[u8]) -> bool {
93        // FIXME(const-hack): This implementation can be reverted when
94        // `core::iter::zip` is allowed in const. The original implementation:
95        //  self.len() == other.len() && iter::zip(self, other).all(|(a, b)| a.eq_ignore_ascii_case(b))
96        let mut a = self;
97        let mut b = other;
98
99        while let ([first_a, rest_a @ ..], [first_b, rest_b @ ..]) = (a, b) {
100            if first_a.eq_ignore_ascii_case(first_b) {
101                a = rest_a;
102                b = rest_b;
103            } else {
104                return false;
105            }
106        }
107
108        true
109    }
110
111    /// Optimized version of `eq_ignore_ascii_case` to process chunks at a time.
112    ///
113    /// Platforms that have SIMD instructions may benefit from this
114    /// implementation over `eq_ignore_ascii_case_simple`.
115    ///
116    /// # Invariants
117    ///
118    /// The caller must guarantee that the slices are equal in length, and the
119    /// slice lengths are greater than or equal to `N` bytes.
120    #[cfg(any(
121        all(target_arch = "x86_64", target_feature = "sse2"),
122        all(target_arch = "aarch64", target_feature = "neon")
123    ))]
124    #[inline]
125    const fn eq_ignore_ascii_case_chunks<const N: usize>(&self, other: &[u8]) -> bool {
126        // FIXME(const-hack): The while-loops that follow should be replaced by
127        // for-loops when available in const.
128
129        let (self_chunks, self_rem) = self.as_chunks::<N>();
130        let (other_chunks, _) = other.as_chunks::<N>();
131
132        // Branchless check to encourage auto-vectorization
133        #[inline(always)]
134        const fn eq_ignore_ascii_inner<const L: usize>(lhs: &[u8; L], rhs: &[u8; L]) -> bool {
135            let mut equal_ascii = true;
136            let mut j = 0;
137            while j < L {
138                equal_ascii &= lhs[j].eq_ignore_ascii_case(&rhs[j]);
139                j += 1;
140            }
141
142            equal_ascii
143        }
144
145        // Process the chunks, returning early if an inequality is found
146        let mut i = 0;
147        while i < self_chunks.len() && i < other_chunks.len() {
148            if !eq_ignore_ascii_inner(&self_chunks[i], &other_chunks[i]) {
149                return false;
150            }
151            i += 1;
152        }
153
154        // Check the length invariant which is necessary for the tail-handling
155        // logic to be correct. This should have been upheld by the caller,
156        // otherwise lengths less than N will compare as true without any
157        // checking.
158        if true {
    if !(self.len() >= N) {
        crate::panicking::panic("assertion failed: self.len() >= N")
    };
};debug_assert!(self.len() >= N);
159
160        // If there are remaining tails, load the last N bytes in the slices to
161        // avoid falling back to per-byte checking.
162        if !self_rem.is_empty() {
163            if let (Some(a_rem), Some(b_rem)) = (self.last_chunk::<N>(), other.last_chunk::<N>()) {
164                if !eq_ignore_ascii_inner(a_rem, b_rem) {
165                    return false;
166                }
167            }
168        }
169
170        true
171    }
172
173    /// Converts this slice to its ASCII upper case equivalent in-place.
174    ///
175    /// ASCII letters 'a' to 'z' are mapped to 'A' to 'Z',
176    /// but non-ASCII letters are unchanged.
177    ///
178    /// To return a new uppercased value without modifying the existing one, use
179    /// [`to_ascii_uppercase`].
180    ///
181    /// [`to_ascii_uppercase`]: #method.to_ascii_uppercase
182    #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
183    #[rustc_const_stable(feature = "const_make_ascii", since = "1.84.0")]
184    #[inline]
185    pub const fn make_ascii_uppercase(&mut self) {
186        // FIXME(const-hack): We would like to simply iterate using `for` loops but this isn't currently allowed in constant expressions.
187        let mut i = 0;
188        while i < self.len() {
189            let byte = &mut self[i];
190            byte.make_ascii_uppercase();
191            i += 1;
192        }
193    }
194
195    /// Converts this slice to its ASCII lower case equivalent in-place.
196    ///
197    /// ASCII letters 'A' to 'Z' are mapped to 'a' to 'z',
198    /// but non-ASCII letters are unchanged.
199    ///
200    /// To return a new lowercased value without modifying the existing one, use
201    /// [`to_ascii_lowercase`].
202    ///
203    /// [`to_ascii_lowercase`]: #method.to_ascii_lowercase
204    #[stable(feature = "ascii_methods_on_intrinsics", since = "1.23.0")]
205    #[rustc_const_stable(feature = "const_make_ascii", since = "1.84.0")]
206    #[inline]
207    pub const fn make_ascii_lowercase(&mut self) {
208        // FIXME(const-hack): We would like to simply iterate using `for` loops but this isn't currently allowed in constant expressions.
209        let mut i = 0;
210        while i < self.len() {
211            let byte = &mut self[i];
212            byte.make_ascii_lowercase();
213            i += 1;
214        }
215    }
216
217    /// Returns an iterator that produces an escaped version of this slice,
218    /// treating it as an ASCII string.
219    ///
220    /// # Examples
221    ///
222    /// ```
223    /// let s = b"0\t\r\n'\"\\\x9d";
224    /// let escaped = s.escape_ascii().to_string();
225    /// assert_eq!(escaped, "0\\t\\r\\n\\'\\\"\\\\\\x9d");
226    /// ```
227    #[must_use = "this returns the escaped bytes as an iterator, \
228                  without modifying the original"]
229    #[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
230    pub fn escape_ascii(&self) -> EscapeAscii<'_> {
231        EscapeAscii { inner: self.iter().flat_map(EscapeByte) }
232    }
233
234    /// Returns a byte slice with leading ASCII whitespace bytes removed.
235    ///
236    /// 'Whitespace' refers to the definition used by
237    /// [`u8::is_ascii_whitespace`]. Importantly, this definition excludes
238    /// the `\0x0B` byte even though it has the Unicode [`White_Space`] property
239    /// and is removed by [`str::trim_start`].
240    ///
241    /// [`White_Space`]: https://www.unicode.org/reports/tr44/#White_Space
242    ///
243    /// # Examples
244    ///
245    /// ```
246    /// assert_eq!(b" \t hello world\n".trim_ascii_start(), b"hello world\n");
247    /// assert_eq!(b"  ".trim_ascii_start(), b"");
248    /// assert_eq!(b"".trim_ascii_start(), b"");
249    /// ```
250    #[stable(feature = "byte_slice_trim_ascii", since = "1.80.0")]
251    #[rustc_const_stable(feature = "byte_slice_trim_ascii", since = "1.80.0")]
252    #[inline]
253    pub const fn trim_ascii_start(&self) -> &[u8] {
254        let mut bytes = self;
255        // Note: A pattern matching based approach (instead of indexing) allows
256        // making the function const.
257        while let [first, rest @ ..] = bytes {
258            if first.is_ascii_whitespace() {
259                bytes = rest;
260            } else {
261                break;
262            }
263        }
264        bytes
265    }
266
267    /// Returns a byte slice with trailing ASCII whitespace bytes removed.
268    ///
269    /// 'Whitespace' refers to the definition used by
270    /// [`u8::is_ascii_whitespace`]. Importantly, this definition excludes
271    /// the `\0x0B` byte even though it has the Unicode [`White_Space`] property
272    /// and is removed by [`str::trim_end`].
273    ///
274    /// [`White_Space`]: https://www.unicode.org/reports/tr44/#White_Space
275    ///
276    /// # Examples
277    ///
278    /// ```
279    /// assert_eq!(b"\r hello world\n ".trim_ascii_end(), b"\r hello world");
280    /// assert_eq!(b"  ".trim_ascii_end(), b"");
281    /// assert_eq!(b"".trim_ascii_end(), b"");
282    /// ```
283    #[stable(feature = "byte_slice_trim_ascii", since = "1.80.0")]
284    #[rustc_const_stable(feature = "byte_slice_trim_ascii", since = "1.80.0")]
285    #[inline]
286    pub const fn trim_ascii_end(&self) -> &[u8] {
287        let mut bytes = self;
288        // Note: A pattern matching based approach (instead of indexing) allows
289        // making the function const.
290        while let [rest @ .., last] = bytes {
291            if last.is_ascii_whitespace() {
292                bytes = rest;
293            } else {
294                break;
295            }
296        }
297        bytes
298    }
299
300    /// Returns a byte slice with leading and trailing ASCII whitespace bytes
301    /// removed.
302    ///
303    /// 'Whitespace' refers to the definition used by
304    /// [`u8::is_ascii_whitespace`]. Importantly, this definition excludes
305    /// the `\0x0B` byte even though it has the Unicode [`White_Space`] property
306    /// and is removed by [`str::trim`].
307    ///
308    /// [`White_Space`]: https://www.unicode.org/reports/tr44/#White_Space
309    ///
310    /// # Examples
311    ///
312    /// ```
313    /// assert_eq!(b"\r hello world\n ".trim_ascii(), b"hello world");
314    /// assert_eq!(b"  ".trim_ascii(), b"");
315    /// assert_eq!(b"".trim_ascii(), b"");
316    /// ```
317    #[stable(feature = "byte_slice_trim_ascii", since = "1.80.0")]
318    #[rustc_const_stable(feature = "byte_slice_trim_ascii", since = "1.80.0")]
319    #[inline]
320    pub const fn trim_ascii(&self) -> &[u8] {
321        self.trim_ascii_start().trim_ascii_end()
322    }
323
324    /// Splits a byte slice by ASCII whitespace.
325    ///
326    /// The returned iterator yields byte slices that are subslices of the
327    /// original byte slice, separated by any amount of ASCII whitespace.
328    ///
329    /// This uses the same definition as [`u8::is_ascii_whitespace`].
330    ///
331    /// # Examples
332    ///
333    /// Basic usage:
334    ///
335    /// ```
336    /// #![feature(u8_split_ascii_whitespace)]
337    ///
338    /// let mut iter = b"A few words".split_ascii_whitespace();
339    ///
340    /// assert_eq!(Some(&b"A"[..]), iter.next());
341    /// assert_eq!(Some(&b"few"[..]), iter.next());
342    /// assert_eq!(Some(&b"words"[..]), iter.next());
343    ///
344    /// assert_eq!(None, iter.next());
345    /// ```
346    ///
347    /// Various kinds of ASCII whitespace are considered
348    /// (see [`u8::is_ascii_whitespace`]):
349    ///
350    /// ```
351    /// #![feature(u8_split_ascii_whitespace)]
352    ///
353    /// let mut iter = b" Mary   had\ta little  \n\t lamb".split_ascii_whitespace();
354    ///
355    /// assert_eq!(Some(&b"Mary"[..]), iter.next());
356    /// assert_eq!(Some(&b"had"[..]), iter.next());
357    /// assert_eq!(Some(&b"a"[..]), iter.next());
358    /// assert_eq!(Some(&b"little"[..]), iter.next());
359    /// assert_eq!(Some(&b"lamb"[..]), iter.next());
360    ///
361    /// assert_eq!(None, iter.next());
362    /// ```
363    ///
364    /// If the byte slice is empty or contains only ASCII whitespace, the iterator
365    /// yields no byte slices:
366    ///
367    /// ```
368    /// #![feature(u8_split_ascii_whitespace)]
369    ///
370    /// assert_eq!(b"".split_ascii_whitespace().next(), None);
371    /// assert_eq!(b"   ".split_ascii_whitespace().next(), None);
372    /// ```
373    #[must_use = "this returns the split byte slice as an iterator, without modifying the original"]
374    #[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
375    #[inline]
376    pub fn split_ascii_whitespace(&self) -> SplitAsciiWhitespace<'_> {
377        let inner = self.split(IsAsciiWhitespace).filter(BytesIsNotEmpty);
378        SplitAsciiWhitespace { inner }
379    }
380}
381
382/// An iterator over the non-ASCII-whitespace subslices of a byte slice,
383/// separated by any amount of ASCII whitespace.
384///
385/// This struct is created by the [`split_ascii_whitespace`] method on [`[u8]`][byteslice].
386/// See its documentation for more.
387///
388/// [`split_ascii_whitespace`]: slice::split_ascii_whitespace
389/// [byteslice]: prim@slice
390#[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
391#[derive(#[automatically_derived]
#[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
impl<'a> crate::clone::Clone for SplitAsciiWhitespace<'a> {
    #[inline]
    fn clone(&self) -> SplitAsciiWhitespace<'a> {
        SplitAsciiWhitespace {
            inner: crate::clone::Clone::clone(&self.inner),
        }
    }
}Clone, #[automatically_derived]
#[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
impl<'a> crate::fmt::Debug for SplitAsciiWhitespace<'a> {
    #[inline]
    fn fmt(&self, f: &mut crate::fmt::Formatter) -> crate::fmt::Result {
        crate::fmt::Formatter::debug_struct_field1_finish(f,
            "SplitAsciiWhitespace", "inner", &&self.inner)
    }
}Debug)]
392pub struct SplitAsciiWhitespace<'a> {
393    pub(crate) inner: Filter<Split<'a, u8, IsAsciiWhitespace>, BytesIsNotEmpty>,
394}
395
396#[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
397impl<'a> Iterator for SplitAsciiWhitespace<'a> {
398    type Item = &'a [u8];
399
400    #[inline]
401    fn next(&mut self) -> Option<&'a [u8]> {
402        self.inner.next()
403    }
404
405    #[inline]
406    fn size_hint(&self) -> (usize, Option<usize>) {
407        self.inner.size_hint()
408    }
409
410    #[inline]
411    fn last(mut self) -> Option<&'a [u8]> {
412        self.next_back()
413    }
414}
415
416#[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
417impl<'a> DoubleEndedIterator for SplitAsciiWhitespace<'a> {
418    #[inline]
419    fn next_back(&mut self) -> Option<&'a [u8]> {
420        self.inner.next_back()
421    }
422}
423
424#[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
425impl FusedIterator for SplitAsciiWhitespace<'_> {}
426
427impl<'a> SplitAsciiWhitespace<'a> {
428    /// Returns remainder of the split slice.
429    ///
430    /// If the iterator is empty, returns `None`.
431    ///
432    /// # Examples
433    ///
434    /// ```
435    /// #![feature(u8_split_ascii_whitespace)]
436    ///
437    /// let mut split = b"Mary had a little lamb".split_ascii_whitespace();
438    /// assert_eq!(split.remainder(), Some(b"Mary had a little lamb".as_slice()));
439    ///
440    /// split.next();
441    /// assert_eq!(split.remainder(), Some(b"had a little lamb".as_slice()));
442    ///
443    /// split.by_ref().for_each(drop);
444    /// assert_eq!(split.remainder(), None);
445    /// ```
446    #[inline]
447    #[must_use]
448    // This is also blocked on: https://github.com/rust-lang/rust/issues/77998
449    #[unstable(feature = "u8_split_ascii_whitespace", issue = "147878")]
450    pub fn remainder(&self) -> Option<&'a [u8]> {
451        if self.inner.iter.finished {
452            return None;
453        }
454
455        Some(self.inner.iter.v)
456    }
457}
458
459struct EscapeByte;
#[automatically_derived]
impl crate::clone::Clone for EscapeByte {
    #[inline]
    fn clone(&self) -> EscapeByte { EscapeByte }
}
impl Fn<(&u8,)> for EscapeByte {
    #[inline]
    extern "rust-call" fn call(&self, (byte,): (&u8,))
        -> ascii::EscapeDefault {
        { ascii::escape_default(*byte) }
    }
}
impl FnMut<(&u8,)> for EscapeByte {
    #[inline]
    extern "rust-call" fn call_mut(&mut self, (byte,): (&u8,))
        -> ascii::EscapeDefault {
        Fn::call(&*self, (byte,))
    }
}
impl FnOnce<(&u8,)> for EscapeByte {
    type Output = ascii::EscapeDefault;
    #[inline]
    extern "rust-call" fn call_once(self, (byte,): (&u8,))
        -> ascii::EscapeDefault {
        Fn::call(&self, (byte,))
    }
}impl_fn_for_zst! {
460    #[derive(Clone)]
461    struct EscapeByte impl Fn = |byte: &u8| -> ascii::EscapeDefault {
462        ascii::escape_default(*byte)
463    };
464}
465
466/// An iterator over the escaped version of a byte slice.
467///
468/// This `struct` is created by the [`slice::escape_ascii`] method. See its
469/// documentation for more information.
470#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
471#[derive(#[automatically_derived]
#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
impl<'a> crate::clone::Clone for EscapeAscii<'a> {
    #[inline]
    fn clone(&self) -> EscapeAscii<'a> {
        EscapeAscii { inner: crate::clone::Clone::clone(&self.inner) }
    }
}Clone)]
472#[must_use = "iterators are lazy and do nothing unless consumed"]
473pub struct EscapeAscii<'a> {
474    inner: iter::FlatMap<super::Iter<'a, u8>, ascii::EscapeDefault, EscapeByte>,
475}
476
477#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
478impl<'a> iter::Iterator for EscapeAscii<'a> {
479    type Item = u8;
480    #[inline]
481    fn next(&mut self) -> Option<u8> {
482        self.inner.next()
483    }
484    #[inline]
485    fn size_hint(&self) -> (usize, Option<usize>) {
486        self.inner.size_hint()
487    }
488    #[inline]
489    fn try_fold<Acc, Fold, R>(&mut self, init: Acc, fold: Fold) -> R
490    where
491        Fold: FnMut(Acc, Self::Item) -> R,
492        R: ops::Try<Output = Acc>,
493    {
494        self.inner.try_fold(init, fold)
495    }
496    #[inline]
497    fn fold<Acc, Fold>(self, init: Acc, fold: Fold) -> Acc
498    where
499        Fold: FnMut(Acc, Self::Item) -> Acc,
500    {
501        self.inner.fold(init, fold)
502    }
503    #[inline]
504    fn last(mut self) -> Option<u8> {
505        self.next_back()
506    }
507}
508
509#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
510impl<'a> iter::DoubleEndedIterator for EscapeAscii<'a> {
511    fn next_back(&mut self) -> Option<u8> {
512        self.inner.next_back()
513    }
514}
515#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
516impl<'a> iter::FusedIterator for EscapeAscii<'a> {}
517#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
518impl<'a> fmt::Display for EscapeAscii<'a> {
519    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
520        // disassemble iterator, including front/back parts of flatmap in case it has been partially consumed
521        let (front, slice, back) = self.clone().inner.into_parts();
522        let front = front.unwrap_or(EscapeDefault::empty());
523        let mut bytes = slice.unwrap_or_default().as_slice();
524        let back = back.unwrap_or(EscapeDefault::empty());
525
526        // usually empty, so the formatter won't have to do any work
527        for byte in front {
528            f.write_char(byte as char)?;
529        }
530
531        fn needs_escape(b: u8) -> bool {
532            b > 0x7E || b < 0x20 || b == b'\\' || b == b'\'' || b == b'"'
533        }
534
535        while !bytes.is_empty() {
536            // fast path for the printable, non-escaped subset of ascii
537            let prefix = bytes.iter().take_while(|&&b| !needs_escape(b)).count();
538            // SAFETY: prefix length was derived by counting bytes in the same splice, so it's in-bounds
539            let (prefix, remainder) = unsafe { bytes.split_at_unchecked(prefix) };
540            // SAFETY: prefix is a valid utf8 sequence, as it's a subset of ASCII
541            let prefix = unsafe { crate::str::from_utf8_unchecked(prefix) };
542
543            f.write_str(prefix)?; // the fast part
544
545            bytes = remainder;
546
547            if let Some(&b) = bytes.first() {
548                // guaranteed to be non-empty, better to write it as a str
549                fmt::Display::fmt(&ascii::escape_default(b), f)?;
550                bytes = &bytes[1..];
551            }
552        }
553
554        // also usually empty
555        for byte in back {
556            f.write_char(byte as char)?;
557        }
558        Ok(())
559    }
560}
561#[stable(feature = "inherent_ascii_escape", since = "1.60.0")]
562impl<'a> fmt::Debug for EscapeAscii<'a> {
563    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
564        f.debug_struct("EscapeAscii").finish_non_exhaustive()
565    }
566}
567
568/// ASCII test *without* the chunk-at-a-time optimizations.
569///
570/// This is carefully structured to produce nice small code -- it's smaller in
571/// `-O` than what the "obvious" ways produces under `-C opt-level=s`.  If you
572/// touch it, be sure to run (and update if needed) the assembly test.
573#[unstable(feature = "str_internals", issue = "none")]
574#[doc(hidden)]
575#[inline]
576pub const fn is_ascii_simple(mut bytes: &[u8]) -> bool {
577    while let [rest @ .., last] = bytes {
578        if !last.is_ascii() {
579            break;
580        }
581        bytes = rest;
582    }
583    bytes.is_empty()
584}
585
586/// Optimized ASCII test that will use usize-at-a-time operations instead of
587/// byte-at-a-time operations (when possible).
588///
589/// The algorithm we use here is pretty simple. If `s` is too short, we just
590/// check each byte and be done with it. Otherwise:
591///
592/// - Read the first word with an unaligned load.
593/// - Align the pointer, read subsequent words until end with aligned loads.
594/// - Read the last `usize` from `s` with an unaligned load.
595///
596/// If any of these loads produces something for which `contains_nonascii`
597/// (above) returns true, then we know the answer is false.
598#[cfg(not(any(
599    all(target_arch = "x86_64", target_feature = "sse2"),
600    all(target_arch = "loongarch64", target_feature = "lsx"),
601    all(target_arch = "aarch64", target_feature = "neon")
602)))]
603#[inline]
604#[rustc_allow_const_fn_unstable(const_eval_select)] // fallback impl has same behavior
605const fn is_ascii(s: &[u8]) -> bool {
606    // The runtime version behaves the same as the compiletime version, it's
607    // just more optimized.
608    const_eval_select!(
609        @capture { s: &[u8] } -> bool:
610        if const {
611            is_ascii_simple(s)
612        } else {
613            /// Returns `true` if any byte in the word `v` is nonascii (>= 128). Snarfed
614            /// from `../str/mod.rs`, which does something similar for utf8 validation.
615            const fn contains_nonascii(v: usize) -> bool {
616                const NONASCII_MASK: usize = usize::repeat_u8(0x80);
617                (NONASCII_MASK & v) != 0
618            }
619
620            const USIZE_SIZE: usize = size_of::<usize>();
621
622            let len = s.len();
623            let align_offset = s.as_ptr().align_offset(USIZE_SIZE);
624
625            // If we wouldn't gain anything from the word-at-a-time implementation, fall
626            // back to a scalar loop.
627            //
628            // We also do this for architectures where `size_of::<usize>()` isn't
629            // sufficient alignment for `usize`, because it's a weird edge case.
630            if len < USIZE_SIZE || len < align_offset || USIZE_SIZE < align_of::<usize>() {
631                return is_ascii_simple(s);
632            }
633
634            // We always read the first word unaligned, which means `align_offset` is
635            // 0, we'd read the same value again for the aligned read.
636            let offset_to_aligned = if align_offset == 0 { USIZE_SIZE } else { align_offset };
637
638            let start = s.as_ptr();
639            // SAFETY: We verify `len < USIZE_SIZE` above.
640            let first_word = unsafe { (start as *const usize).read_unaligned() };
641
642            if contains_nonascii(first_word) {
643                return false;
644            }
645            // We checked this above, somewhat implicitly. Note that `offset_to_aligned`
646            // is either `align_offset` or `USIZE_SIZE`, both of are explicitly checked
647            // above.
648            debug_assert!(offset_to_aligned <= len);
649
650            // SAFETY: word_ptr is the (properly aligned) usize ptr we use to read the
651            // middle chunk of the slice.
652            let mut word_ptr = unsafe { start.add(offset_to_aligned) as *const usize };
653
654            // `byte_pos` is the byte index of `word_ptr`, used for loop end checks.
655            let mut byte_pos = offset_to_aligned;
656
657            // Paranoia check about alignment, since we're about to do a bunch of
658            // unaligned loads. In practice this should be impossible barring a bug in
659            // `align_offset` though.
660            // While this method is allowed to spuriously fail in CTFE, if it doesn't
661            // have alignment information it should have given a `usize::MAX` for
662            // `align_offset` earlier, sending things through the scalar path instead of
663            // this one, so this check should pass if it's reachable.
664            debug_assert!(word_ptr.is_aligned_to(align_of::<usize>()));
665
666            // Read subsequent words until the last aligned word, excluding the last
667            // aligned word by itself to be done in tail check later, to ensure that
668            // tail is always one `usize` at most to extra branch `byte_pos == len`.
669            while byte_pos < len - USIZE_SIZE {
670                // Sanity check that the read is in bounds
671                debug_assert!(byte_pos + USIZE_SIZE <= len);
672                // And that our assumptions about `byte_pos` hold.
673                debug_assert!(word_ptr.cast::<u8>() == start.wrapping_add(byte_pos));
674
675                // SAFETY: We know `word_ptr` is properly aligned (because of
676                // `align_offset`), and we know that we have enough bytes between `word_ptr` and the end
677                let word = unsafe { word_ptr.read() };
678                if contains_nonascii(word) {
679                    return false;
680                }
681
682                byte_pos += USIZE_SIZE;
683                // SAFETY: We know that `byte_pos <= len - USIZE_SIZE`, which means that
684                // after this `add`, `word_ptr` will be at most one-past-the-end.
685                word_ptr = unsafe { word_ptr.add(1) };
686            }
687
688            // Sanity check to ensure there really is only one `usize` left. This should
689            // be guaranteed by our loop condition.
690            debug_assert!(byte_pos <= len && len - byte_pos <= USIZE_SIZE);
691
692            // SAFETY: This relies on `len >= USIZE_SIZE`, which we check at the start.
693            let last_word = unsafe { (start.add(len - USIZE_SIZE) as *const usize).read_unaligned() };
694
695            !contains_nonascii(last_word)
696        }
697    )
698}
699
700/// Chunk size for SSE2 vectorized ASCII checking (4x 16-byte loads).
701#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
702const SSE2_CHUNK_SIZE: usize = 64;
703
704#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
705#[inline]
706fn is_ascii_sse2(bytes: &[u8]) -> bool {
707    use crate::arch::x86_64::{__m128i, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128};
708
709    let (chunks, rest) = bytes.as_chunks::<SSE2_CHUNK_SIZE>();
710
711    for chunk in chunks {
712        let ptr = chunk.as_ptr();
713        // SAFETY: chunk is 64 bytes. SSE2 is baseline on x86_64.
714        let mask = unsafe {
715            let a1 = _mm_loadu_si128(ptr as *const __m128i);
716            let a2 = _mm_loadu_si128(ptr.add(16) as *const __m128i);
717            let b1 = _mm_loadu_si128(ptr.add(32) as *const __m128i);
718            let b2 = _mm_loadu_si128(ptr.add(48) as *const __m128i);
719            // OR all chunks - if any byte has high bit set, combined will too.
720            let combined = _mm_or_si128(_mm_or_si128(a1, a2), _mm_or_si128(b1, b2));
721            // Create a mask from the MSBs of each byte.
722            // If any byte is >= 128, its MSB is 1, so the mask will be non-zero.
723            _mm_movemask_epi8(combined)
724        };
725        if mask != 0 {
726            return false;
727        }
728    }
729
730    // Handle remaining bytes
731    rest.iter().all(|b| b.is_ascii())
732}
733
734/// Chunk size for NEON vectorized ASCII checking (4x 16-byte loads).
735#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
736const NEON_CHUNK_SIZE: usize = 64;
737
738/// Width of a single NEON vector, used to vectorize the tail left over by the
739/// unrolled `NEON_CHUNK_SIZE` loop.
740#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
741const NEON_VECTOR_SIZE: usize = 16;
742
743#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
744#[inline]
745fn is_ascii_neon(bytes: &[u8]) -> bool {
746    use crate::arch::aarch64::{vld1q_u8, vmaxvq_u8, vorrq_u8};
747
748    let (chunks, rest) = bytes.as_chunks::<NEON_CHUNK_SIZE>();
749
750    for chunk in chunks {
751        let ptr = chunk.as_ptr();
752        // SAFETY: chunk is 64 bytes, and `vld1q_u8` has no alignment requirement.
753        let max = unsafe {
754            let a1 = vld1q_u8(ptr);
755            let a2 = vld1q_u8(ptr.add(16));
756            let b1 = vld1q_u8(ptr.add(32));
757            let b2 = vld1q_u8(ptr.add(48));
758            // OR all chunks - if any byte has high bit set, combined will too.
759            let combined = vorrq_u8(vorrq_u8(a1, a2), vorrq_u8(b1, b2));
760            // `vmaxvq_u8` is a horizontal reduction with a longer latency than
761            // `vorrq_u8`, so it runs once per 64 bytes rather than once per load.
762            vmaxvq_u8(combined)
763        };
764        if max >= 128 {
765            return false;
766        }
767    }
768
769    // The unrolled loop above leaves up to 63 bytes, so sweep those a vector at
770    // a time before falling back to a byte-at-a-time check.
771    let (vectors, rest) = rest.as_chunks::<NEON_VECTOR_SIZE>();
772
773    for vector in vectors {
774        // SAFETY: vector is 16 bytes, and `vld1q_u8` has no alignment requirement.
775        let max = unsafe { vmaxvq_u8(vld1q_u8(vector.as_ptr())) };
776        if max >= 128 {
777            return false;
778        }
779    }
780
781    // Handle remaining bytes
782    rest.iter().all(|b| b.is_ascii())
783}
784
785/// Uses explicit SIMD intrinsics to prevent LLVM from auto-vectorizing with
786/// broken code (e.g., AVX-512 on x86-64 that extracts mask bits one-by-one).
787#[cfg(any(
788    all(target_arch = "x86_64", target_feature = "sse2"),
789    all(target_arch = "aarch64", target_feature = "neon")
790))]
791#[inline]
792#[rustc_allow_const_fn_unstable(const_eval_select)]
793const fn is_ascii(bytes: &[u8]) -> bool {
794    const USIZE_SIZE: usize = size_of::<usize>();
795    const NONASCII_MASK: usize = usize::MAX / 255 * 0x80;
796
797    #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
798    const SIMD_MIN_LEN: usize = SSE2_CHUNK_SIZE;
799    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
800    const SIMD_MIN_LEN: usize = NEON_CHUNK_SIZE;
801
802    {
    #[inline]
    fn runtime(bytes: &[u8]) -> bool {
        {
            if bytes.len() < SIMD_MIN_LEN {
                let (chunks, remainder) = bytes.as_chunks::<USIZE_SIZE>();
                for chunk in chunks {
                    let word = usize::from_ne_bytes(*chunk);
                    if (word & NONASCII_MASK) != 0 { return false; }
                }
                return remainder.iter().all(|b| b.is_ascii());
            }
            { is_ascii_sse2(bytes) }
        }
    }
    #[inline]
    const fn compiletime(bytes: &[u8]) -> bool {
        let _ = bytes;
        { is_ascii_simple(bytes) }
    }
    const_eval_select((bytes,), compiletime, runtime)
}const_eval_select!(
803        @capture { bytes: &[u8] } -> bool:
804        if const {
805            is_ascii_simple(bytes)
806        } else {
807            // For small inputs, use usize-at-a-time processing to avoid SSE2 call overhead.
808            if bytes.len() < SIMD_MIN_LEN {
809                let (chunks, remainder) = bytes.as_chunks::<USIZE_SIZE>();
810                for chunk in chunks {
811                    let word = usize::from_ne_bytes(*chunk);
812                    if (word & NONASCII_MASK) != 0 {
813                        return false;
814                    }
815                }
816                return remainder.iter().all(|b| b.is_ascii());
817            }
818
819            #[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
820            { is_ascii_sse2(bytes) }
821            #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
822            { is_ascii_neon(bytes) }
823        }
824    )
825}
826
827/// ASCII test optimized to use the `vmskltz.b` instruction on `loongarch64`.
828///
829/// Other platforms are not likely to benefit from this code structure, so they
830/// use SWAR techniques to test for ASCII in `usize`-sized chunks.
831#[cfg(all(target_arch = "loongarch64", target_feature = "lsx"))]
832#[inline]
833const fn is_ascii(bytes: &[u8]) -> bool {
834    // Process chunks of 32 bytes at a time in the fast path to enable
835    // auto-vectorization and use of `vmskltz.b`. Two 128-bit vector registers
836    // can be OR'd together and then the resulting vector can be tested for
837    // non-ASCII bytes.
838    const CHUNK_SIZE: usize = 32;
839
840    let mut i = 0;
841
842    while i + CHUNK_SIZE <= bytes.len() {
843        let chunk_end = i + CHUNK_SIZE;
844
845        // Get LLVM to produce a `vmskltz.b` instruction on loongarch64 which
846        // creates a mask from the most significant bit of each byte.
847        // ASCII bytes are less than 128 (0x80), so their most significant
848        // bit is unset.
849        let mut count = 0;
850        while i < chunk_end {
851            count += bytes[i].is_ascii() as u8;
852            i += 1;
853        }
854
855        // All bytes should be <= 127 so count is equal to chunk size.
856        if count != CHUNK_SIZE as u8 {
857            return false;
858        }
859    }
860
861    // Process the remaining `bytes.len() % N` bytes.
862    let mut is_ascii = true;
863    while i < bytes.len() {
864        is_ascii &= bytes[i].is_ascii();
865        i += 1;
866    }
867
868    is_ascii
869}
core/slice/ascii.rs

core/slice/
ascii.rs
