alloc/io/copy/
generic.rs

1use core::mem::MaybeUninit;
2
3#[cfg(not(no_global_oom_handling))]
4use crate::collections::VecDeque;
5use crate::io::{BorrowedBuf, BufReader, BufWriter, DEFAULT_BUF_SIZE, Read, Result, Write};
6use crate::vec::Vec;
7#[cfg_attr(
8    no_global_oom_handling,
9    expect(unused_imports, reason = "only required for VecDeque specialization")
10)]
11use crate::{alloc::Allocator, io::IoSlice};
12
13/// The userspace read-write-loop implementation of `io::copy` that is used when
14/// OS-specific specializations for copy offloading are not available or not applicable.
15///
16/// This function is able to perform a mild amount of specialization based on
17/// the size of the `reader` and `writer` buffers, if they have any.
18///
19/// * If `reader`'s buffer is large enough ([`>=DEFAULT_BUF_SIZE`](DEFAULT_BUF_SIZE)),
20///   _and_ it is larger than `writer`'s buffer, copying will be controlled by `R`.
21/// * Otherwise, copying will be controlled by `writer`.
22///
23/// Currently, `[u8]`, `Vec<u8>`, `VecDeque<u8>`, `BufReader<T>` and `BufWriter<T>`
24/// are specialized with.
25pub(super) fn generic_copy<R: ?Sized, W: ?Sized>(reader: &mut R, writer: &mut W) -> Result<u64>
26where
27    R: Read,
28    W: Write,
29{
30    let read_priority = BufferedReaderSpec::buffer_priority(reader);
31    let write_priority = BufferedWriterSpec::buffer_priority(writer);
32
33    if read_priority >= DEFAULT_BUF_SIZE && read_priority >= write_priority {
34        return BufferedReaderSpec::copy_to(reader, writer);
35    }
36
37    BufferedWriterSpec::copy_from(writer, reader)
38}
39
40/// This is used by [`generic_copy`] to decide whether to use [`BufferedReaderSpec::copy_to`]
41/// or [`BufferedWriterSpec::copy_from`].
42type BufferPriority = usize;
43
44/// Unbuffered readers and writers have the lowest priority.
45const UNBUFFERED: BufferPriority = BufferPriority::MIN;
46
47/// Readers and writers with their entire contents have the highest priority.
48const IN_MEMORY: BufferPriority = BufferPriority::MAX;
49
50/// Specialization of the read-write loop in [`generic_copy`] that reuses the
51/// internal buffer of a [`BufReader`]. If there's no buffer then the writer side
52/// should be used instead.
53trait BufferedReaderSpec {
54    fn buffer_priority(&self) -> BufferPriority;
55
56    fn copy_to(&mut self, to: &mut (impl Write + ?Sized)) -> Result<u64>;
57}
58
59impl<T> BufferedReaderSpec for T
60where
61    Self: Read,
62    T: ?Sized,
63{
64    #[inline]
65    default fn buffer_priority(&self) -> BufferPriority {
66        UNBUFFERED
67    }
68
69    default fn copy_to(&mut self, _to: &mut (impl Write + ?Sized)) -> Result<u64> {
70        {
    ::core::panicking::panic_fmt(format_args!("internal error: entered unreachable code: {0}",
            format_args!("only called from specializations")));
}unreachable!("only called from specializations")
71    }
72}
73
74impl BufferedReaderSpec for &[u8] {
75    fn buffer_priority(&self) -> BufferPriority {
76        IN_MEMORY
77    }
78
79    fn copy_to(&mut self, to: &mut (impl Write + ?Sized)) -> Result<u64> {
80        let len = self.len();
81        to.write_all(self)?;
82        *self = &self[len..];
83        Ok(len as u64)
84    }
85}
86
87#[cfg(not(no_global_oom_handling))]
88impl<A: Allocator> BufferedReaderSpec for VecDeque<u8, A> {
89    fn buffer_priority(&self) -> BufferPriority {
90        IN_MEMORY
91    }
92
93    fn copy_to(&mut self, to: &mut (impl Write + ?Sized)) -> Result<u64> {
94        let len = self.len();
95        let (front, back) = self.as_slices();
96        let bufs = &mut [IoSlice::new(front), IoSlice::new(back)];
97        to.write_all_vectored(bufs)?;
98        self.clear();
99        Ok(len as u64)
100    }
101}
102
103impl<I> BufferedReaderSpec for BufReader<I>
104where
105    Self: Read,
106    I: ?Sized,
107{
108    fn buffer_priority(&self) -> BufferPriority {
109        self.capacity()
110    }
111
112    fn copy_to(&mut self, to: &mut (impl Write + ?Sized)) -> Result<u64> {
113        let mut len = 0;
114
115        loop {
116            // Hack: this relies on `impl Read for BufReader` always calling fill_buf
117            // if the buffer is empty, even for empty slices.
118            // It can't be called directly here since specialization prevents us
119            // from adding I: Read
120            match self.read(&mut []) {
121                Ok(_) => {}
122                Err(e) if e.is_interrupted() => continue,
123                Err(e) => return Err(e),
124            }
125            let buf = self.buffer();
126            if self.buffer().is_empty() {
127                return Ok(len);
128            }
129
130            // In case the writer side is a BufWriter then its write_all
131            // implements an optimization that passes through large
132            // buffers to the underlying writer. That code path is #[cold]
133            // but we're still avoiding redundant memcopies when doing
134            // a copy between buffered inputs and outputs.
135            to.write_all(buf)?;
136            len += buf.len() as u64;
137            self.discard_buffer();
138        }
139    }
140}
141
142/// Specialization of the read-write loop in `generic_copy` that either uses a
143/// stack buffer or reuses the internal buffer of a [`BufWriter`].
144trait BufferedWriterSpec: Write {
145    fn buffer_priority(&self) -> BufferPriority;
146
147    fn copy_from<R: Read + ?Sized>(&mut self, reader: &mut R) -> Result<u64>;
148}
149
150impl<W: Write + ?Sized> BufferedWriterSpec for W {
151    #[inline]
152    default fn buffer_priority(&self) -> BufferPriority {
153        UNBUFFERED
154    }
155
156    default fn copy_from<R: Read + ?Sized>(&mut self, reader: &mut R) -> Result<u64> {
157        // Unlike `BufferedReaderSpec::copy_to`, this _will_ be called as the fallback
158        // when both the reader and writer provide no specialization.
159        stack_buffer_copy(reader, self)
160    }
161}
162
163impl<I: Write + ?Sized> BufferedWriterSpec for BufWriter<I> {
164    fn buffer_priority(&self) -> BufferPriority {
165        self.capacity()
166    }
167
168    fn copy_from<R: Read + ?Sized>(&mut self, reader: &mut R) -> Result<u64> {
169        if self.capacity() < DEFAULT_BUF_SIZE {
170            // Since neither this buffer nor the reader's buffer are large enough,
171            // fall back to the unspecialized implementation.
172            return stack_buffer_copy(reader, self);
173        }
174
175        let mut len = 0;
176        let mut init = false;
177
178        loop {
179            let buf = self.buffer_mut();
180            let mut read_buf: BorrowedBuf<'_, u8> = buf.spare_capacity_mut().into();
181
182            if init {
183                // SAFETY: `init` is only true after `reader` initializes
184                // `read_buf`. See the comment about `flush_buf` below.
185                unsafe { read_buf.set_init() };
186            }
187
188            if read_buf.capacity() >= DEFAULT_BUF_SIZE {
189                let mut cursor = read_buf.unfilled();
190                match reader.read_buf(cursor.reborrow()) {
191                    Ok(()) => {
192                        let bytes_read = cursor.written();
193
194                        if bytes_read == 0 {
195                            return Ok(len);
196                        }
197
198                        init = read_buf.is_init();
199                        len += bytes_read as u64;
200
201                        // SAFETY: BorrowedBuf guarantees all of its filled bytes are init
202                        unsafe { buf.set_len(buf.len() + bytes_read) };
203
204                        // Read again if the buffer still has enough capacity, as BufWriter itself would do
205                        // This will occur if the reader returns short reads
206                    }
207                    Err(ref e) if e.is_interrupted() => {}
208                    Err(e) => return Err(e),
209                }
210            } else {
211                // SAFETY: `flush_buf` will not de-initialize any elements of
212                // the spare capacity so we can remember `init` across this.
213                self.flush_buf()?;
214            }
215        }
216    }
217}
218
219impl BufferedWriterSpec for Vec<u8> {
220    fn buffer_priority(&self) -> BufferPriority {
221        self.capacity() - self.len()
222    }
223
224    fn copy_from<R: Read + ?Sized>(&mut self, reader: &mut R) -> Result<u64> {
225        reader.read_to_end(self).map(|bytes| u64::try_from(bytes).expect("usize overflowed u64"))
226    }
227}
228
229/// Copies from `reader` to `writer` using a stack-allocated buffer (a fixed sized array).
230fn stack_buffer_copy<R: Read + ?Sized, W: Write + ?Sized>(
231    reader: &mut R,
232    writer: &mut W,
233) -> Result<u64> {
234    let buf: &mut [_] = &mut [MaybeUninit::uninit(); DEFAULT_BUF_SIZE];
235    let mut buf: BorrowedBuf<'_, u8> = buf.into();
236
237    let mut len = 0;
238
239    loop {
240        match reader.read_buf(buf.unfilled()) {
241            Ok(()) => {}
242            Err(e) if e.is_interrupted() => continue,
243            Err(e) => return Err(e),
244        };
245
246        if buf.filled().is_empty() {
247            break;
248        }
249
250        len += buf.filled().len() as u64;
251        writer.write_all(buf.filled())?;
252        buf.clear();
253    }
254
255    Ok(len)
256}
alloc/io/copy/generic.rs

alloc/io/copy/
generic.rs
