Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_text/tests/annealer_corpus/bytes_buf.rs

46.7 KiB, 1 run

created by r1870400018:11704, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1use core::mem::{self, ManuallyDrop};
2use core::ops::{Deref, RangeBounds};
3use core::ptr::NonNull;
4use core::{cmp, fmt, hash, ptr, slice};
5
6use alloc::{
7 alloc::{dealloc, Layout},
8 borrow::Borrow,
9 boxed::Box,
10 string::String,
11 vec::Vec,
12};
13
14use crate::buf::IntoIter;
15#[allow(unused)]
16use crate::loom::sync::atomic::AtomicMut;
17use crate::loom::sync::atomic::{AtomicPtr, AtomicUsize, Ordering};
18use crate::{Buf, BytesMut};
19
20/// A cheaply cloneable and sliceable chunk of contiguous memory.
21///
22/// `Bytes` is an efficient container for storing and operating on contiguous
23/// slices of memory. It is intended for use primarily in networking code, but
24/// could have applications elsewhere as well.
25///
26/// `Bytes` values facilitate zero-copy network programming by allowing multiple
27/// `Bytes` objects to point to the same underlying memory.
28///
29/// `Bytes` does not have a single implementation. It is an interface, whose
30/// exact behavior is implemented through dynamic dispatch in several underlying
31/// implementations of `Bytes`.
32///
33/// All `Bytes` implementations must fulfill the following requirements:
34/// - They are cheaply cloneable and thereby shareable between an unlimited amount
35/// of components, for example by modifying a reference count.
36/// - Instances can be sliced to refer to a subset of the original buffer.
37///
38/// ```
39/// use bytes::Bytes;
40///
41/// let mut mem = Bytes::from("Hello world");
42/// let a = mem.slice(0..5);
43///
44/// assert_eq!(a, "Hello");
45///
46/// let b = mem.split_to(6);
47///
48/// assert_eq!(mem, "world");
49/// assert_eq!(b, "Hello ");
50/// ```
51///
52/// # Memory layout
53///
54/// The `Bytes` struct itself is fairly small, limited to 4 `usize` fields used
55/// to track information about which segment of the underlying memory the
56/// `Bytes` handle has access to.
57///
58/// `Bytes` keeps both a pointer to the shared state containing the full memory
59/// slice and a pointer to the start of the region visible by the handle.
60/// `Bytes` also tracks the length of its view into the memory.
61///
62/// # Sharing
63///
64/// `Bytes` contains a vtable, which allows implementations of `Bytes` to define
65/// how sharing/cloning is implemented in detail.
66/// When `Bytes::clone()` is called, `Bytes` will call the vtable function for
67/// cloning the backing storage in order to share it behind multiple `Bytes`
68/// instances.
69///
70/// For `Bytes` implementations which refer to constant memory (e.g. created
71/// via `Bytes::from_static()`) the cloning implementation will be a no-op.
72///
73/// For `Bytes` implementations which point to a reference counted shared storage
74/// (e.g. an `Arc<[u8]>`), sharing will be implemented by increasing the
75/// reference count.
76///
77/// Due to this mechanism, multiple `Bytes` instances may point to the same
78/// shared memory region.
79/// Each `Bytes` instance can point to different sections within that
80/// memory region, and `Bytes` instances may or may not have overlapping views
81/// into the memory.
82///
83/// The following diagram visualizes a scenario where 2 `Bytes` instances make
84/// use of an `Arc`-based backing storage, and provide access to different views:
85///
86/// ```text
87///
88/// Arc ptrs ┌─────────┐
89/// ________________________ / │ Bytes 2 │
90/// / └─────────┘
91/// / ┌───────────┐ | |
92/// |_________/ │ Bytes 1 │ | |
93/// | └───────────┘ | |
94/// | | | ___/ data | tail
95/// | data | tail |/ |
96/// v v v v
97/// ┌─────┬─────┬───────────┬───────────────┬─────┐
98/// │ Arc │ │ │ │ │
99/// └─────┴─────┴───────────┴───────────────┴─────┘
100/// ```
101pub struct Bytes {
102 ptr: *const u8,
103 len: usize,
104 // inlined "trait object"
105 data: AtomicPtr<()>,
106 vtable: &'static Vtable,
107}
108
109// `data` is passed by value (`*mut ()` instead of `&mut AtomicPtr<()>`)
110// when `&mut self` or `self` is consumed.
111// This allows the optimizer to see that the address of the `Bytes` is not
112// captured by the indirect call, enabling further optimizations.
113pub(crate) struct Vtable {
114 /// fn(data, ptr, len)
115 pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes,
116 /// fn(data, ptr, len)
117 ///
118 /// `into_*` consumes the `Bytes`, returning the respective value.
119 pub into_vec: unsafe fn(*mut (), *const u8, usize) -> Vec<u8>,
120 pub into_mut: unsafe fn(*mut (), *const u8, usize) -> BytesMut,
121 /// fn(data)
122 pub is_unique: unsafe fn(&AtomicPtr<()>) -> bool,
123 /// fn(data, ptr, len)
124 pub drop: unsafe fn(*mut (), *const u8, usize),
125}
126
127impl Bytes {
128 /// Creates a new empty `Bytes`.
129 ///
130 /// This will not allocate and the returned `Bytes` handle will be empty.
131 ///
132 /// # Examples
133 ///
134 /// ```
135 /// use bytes::Bytes;
136 ///
137 /// let b = Bytes::new();
138 /// assert_eq!(&b[..], b"");
139 /// ```
140 #[inline]
141 #[cfg(not(all(loom, test)))]
142 pub const fn new() -> Self {
143 // Make it a named const to work around
144 // "unsizing casts are not allowed in const fn"
145 const EMPTY: &[u8] = &[];
146 Bytes::from_static(EMPTY)
147 }
148
149 /// Creates a new empty `Bytes`.
150 #[cfg(all(loom, test))]
151 pub fn new() -> Self {
152 const EMPTY: &[u8] = &[];
153 Bytes::from_static(EMPTY)
154 }
155
156 /// Creates a new `Bytes` from a static slice.
157 ///
158 /// The returned `Bytes` will point directly to the static slice. There is
159 /// no allocating or copying.
160 ///
161 /// # Examples
162 ///
163 /// ```
164 /// use bytes::Bytes;
165 ///
166 /// let b = Bytes::from_static(b"hello");
167 /// assert_eq!(&b[..], b"hello");
168 /// ```
169 #[inline]
170 #[cfg(not(all(loom, test)))]
171 pub const fn from_static(bytes: &'static [u8]) -> Self {
172 Bytes {
173 ptr: bytes.as_ptr(),
174 len: bytes.len(),
175 data: AtomicPtr::new(ptr::null_mut()),
176 vtable: &STATIC_VTABLE,
177 }
178 }
179
180 /// Creates a new `Bytes` from a static slice.
181 #[cfg(all(loom, test))]
182 pub fn from_static(bytes: &'static [u8]) -> Self {
183 Bytes {
184 ptr: bytes.as_ptr(),
185 len: bytes.len(),
186 data: AtomicPtr::new(ptr::null_mut()),
187 vtable: &STATIC_VTABLE,
188 }
189 }
190
191 /// Creates a new `Bytes` with length zero and the given pointer as the address.
192 fn new_empty_with_ptr(ptr: *const u8) -> Self {
193 debug_assert!(!ptr.is_null());
194
195 // Detach this pointer's provenance from whichever allocation it came from, and reattach it
196 // to the provenance of the fake ZST [u8;0] at the same address.
197 let ptr = without_provenance(ptr as usize);
198
199 Bytes {
200 ptr,
201 len: 0,
202 data: AtomicPtr::new(ptr::null_mut()),
203 vtable: &STATIC_VTABLE,
204 }
205 }
206
207 /// Create [Bytes] with a buffer whose lifetime is controlled
208 /// via an explicit owner.
209 ///
210 /// A common use case is to zero-copy construct from mapped memory.
211 ///
212 /// ```
213 /// # struct File;
214 /// #
215 /// # impl File {
216 /// # pub fn open(_: &str) -> Result<Self, ()> {
217 /// # Ok(Self)
218 /// # }
219 /// # }
220 /// #
221 /// # mod memmap2 {
222 /// # pub struct Mmap;
223 /// #
224 /// # impl Mmap {
225 /// # pub unsafe fn map(_file: &super::File) -> Result<Self, ()> {
226 /// # Ok(Self)
227 /// # }
228 /// # }
229 /// #
230 /// # impl AsRef<[u8]> for Mmap {
231 /// # fn as_ref(&self) -> &[u8] {
232 /// # b"buf"
233 /// # }
234 /// # }
235 /// # }
236 /// use bytes::Bytes;
237 /// use memmap2::Mmap;
238 ///
239 /// # fn main() -> Result<(), ()> {
240 /// let file = File::open("upload_bundle.tar.gz")?;
241 /// let mmap = unsafe { Mmap::map(&file) }?;
242 /// let b = Bytes::from_owner(mmap);
243 /// # Ok(())
244 /// # }
245 /// ```
246 ///
247 /// The `owner` will be transferred to the constructed [Bytes] object, which
248 /// will ensure it is dropped once all remaining clones of the constructed
249 /// object are dropped. The owner will then be responsible for dropping the
250 /// specified region of memory as part of its [Drop] implementation.
251 ///
252 /// Note that converting [Bytes] constructed from an owner into a [BytesMut]
253 /// will always create a deep copy of the buffer into newly allocated memory.
254 pub fn from_owner<T>(owner: T) -> Self
255 where
256 T: AsRef<[u8]> + Send + 'static,
257 {
258 // Safety & Miri:
259 // The ownership of `owner` is first transferred to the `Owned` wrapper and `Bytes` object.
260 // This ensures that the owner is pinned in memory, allowing us to call `.as_ref()` safely
261 // since the lifetime of the owner is controlled by the lifetime of the new `Bytes` object,
262 // and the lifetime of the resulting borrowed `&[u8]` matches that of the owner.
263 // Note that this remains safe so long as we only call `.as_ref()` once.
264 //
265 // There are some additional special considerations here:
266 // * We rely on Bytes's Drop impl to clean up memory should `.as_ref()` panic.
267 // * Setting the `ptr` and `len` on the bytes object last (after moving the owner to
268 // Bytes) allows Miri checks to pass since it avoids obtaining the `&[u8]` slice
269 // from a stack-owned Box.
270 // More details on this: https://github.com/tokio-rs/bytes/pull/742/#discussion_r1813375863
271 // and: https://github.com/tokio-rs/bytes/pull/742/#discussion_r1813316032
272
273 let owned = Box::into_raw(Box::new(Owned {
274 ref_cnt: AtomicUsize::new(1),
275 owner,
276 }));
277
278 let mut ret = Bytes {
279 ptr: NonNull::dangling().as_ptr(),
280 len: 0,
281 data: AtomicPtr::new(owned.cast()),
282 vtable: &Owned::<T>::VTABLE,
283 };
284
285 let buf = unsafe { &*owned }.owner.as_ref();
286 ret.ptr = buf.as_ptr();
287 ret.len = buf.len();
288
289 ret
290 }
291
292 /// Returns the number of bytes contained in this `Bytes`.
293 ///
294 /// # Examples
295 ///
296 /// ```
297 /// use bytes::Bytes;
298 ///
299 /// let b = Bytes::from(&b"hello"[..]);
300 /// assert_eq!(b.len(), 5);
301 /// ```
302 #[inline]
303 pub const fn len(&self) -> usize {
304 self.len
305 }
306
307 /// Returns true if the `Bytes` has a length of 0.
308 ///
309 /// # Examples
310 ///
311 /// ```
312 /// use bytes::Bytes;
313 ///
314 /// let b = Bytes::new();
315 /// assert!(b.is_empty());
316 /// ```
317 #[inline]
318 pub const fn is_empty(&self) -> bool {
319 self.len == 0
320 }
321
322 /// Returns true if this is the only reference to the data and
323 /// `Into<BytesMut>` would avoid cloning the underlying buffer.
324 ///
325 /// Always returns false if the data is backed by a [static slice](Bytes::from_static),
326 /// or an [owner](Bytes::from_owner).
327 ///
328 /// The result of this method may be invalidated immediately if another
329 /// thread clones this value while this is being called. Ensure you have
330 /// unique access to this value (`&mut Bytes`) first if you need to be
331 /// certain the result is valid (i.e. for safety reasons).
332 /// # Examples
333 ///
334 /// ```
335 /// use bytes::Bytes;
336 ///
337 /// let a = Bytes::from(vec![1, 2, 3]);
338 /// assert!(a.is_unique());
339 /// let b = a.clone();
340 /// assert!(!a.is_unique());
341 /// ```
342 pub fn is_unique(&self) -> bool {
343 unsafe { (self.vtable.is_unique)(&self.data) }
344 }
345
346 /// Creates `Bytes` instance from slice, by copying it.
347 pub fn copy_from_slice(data: &[u8]) -> Self {
348 data.to_vec().into()
349 }
350
351 /// Returns a slice of self for the provided range.
352 ///
353 /// This will increment the reference count for the underlying memory and
354 /// return a new `Bytes` handle set to the slice.
355 ///
356 /// This operation is `O(1)`.
357 ///
358 /// # Examples
359 ///
360 /// ```
361 /// use bytes::Bytes;
362 ///
363 /// let a = Bytes::from(&b"hello world"[..]);
364 /// let b = a.slice(2..5);
365 ///
366 /// assert_eq!(&b[..], b"llo");
367 /// ```
368 ///
369 /// # Panics
370 ///
371 /// Requires that `begin <= end` and `end <= self.len()`, otherwise slicing
372 /// will panic.
373 pub fn slice(&self, range: impl RangeBounds<usize>) -> Self {
374 let (begin, end) = crate::range(range, self.len());
375
376 if end == begin {
377 return Bytes::new_empty_with_ptr(self.ptr.wrapping_add(begin));
378 }
379
380 let mut ret = self.clone();
381
382 ret.len = end - begin;
383 ret.ptr = unsafe { ret.ptr.add(begin) };
384
385 ret
386 }
387
388 /// Returns a slice of self that is equivalent to the given `subset`.
389 ///
390 /// When processing a `Bytes` buffer with other tools, one often gets a
391 /// `&[u8]` which is in fact a slice of the `Bytes`, i.e. a subset of it.
392 /// This function turns that `&[u8]` into another `Bytes`, as if one had
393 /// called `self.slice()` with the offsets that correspond to `subset`.
394 ///
395 /// This operation is `O(1)`.
396 ///
397 /// # Examples
398 ///
399 /// ```
400 /// use bytes::Bytes;
401 ///
402 /// let bytes = Bytes::from(&b"012345678"[..]);
403 /// let as_slice = bytes.as_ref();
404 /// let subset = &as_slice[2..6];
405 /// let subslice = bytes.slice_ref(&subset);
406 /// assert_eq!(&subslice[..], b"2345");
407 /// ```
408 ///
409 /// # Panics
410 ///
411 /// Requires that the given `sub` slice is in fact contained within the
412 /// `Bytes` buffer; otherwise this function will panic.
413 pub fn slice_ref(&self, subset: &[u8]) -> Self {
414 // Empty slice and empty Bytes may have their pointers reset
415 // so explicitly allow empty slice to be a subslice of any slice.
416 if subset.is_empty() {
417 return Bytes::new();
418 }
419
420 let bytes_p = self.as_ptr() as usize;
421 let bytes_len = self.len();
422
423 let sub_p = subset.as_ptr() as usize;
424 let sub_len = subset.len();
425
426 assert!(
427 sub_p >= bytes_p,
428 "subset pointer ({:p}) is smaller than self pointer ({:p})",
429 subset.as_ptr(),
430 self.as_ptr(),
431 );
432 assert!(
433 sub_p + sub_len <= bytes_p + bytes_len,
434 "subset is out of bounds: self = ({:p}, {}), subset = ({:p}, {})",
435 self.as_ptr(),
436 bytes_len,
437 subset.as_ptr(),
438 sub_len,
439 );
440
441 let sub_offset = sub_p - bytes_p;
442
443 self.slice(sub_offset..(sub_offset + sub_len))
444 }
445
446 /// Splits the bytes into two at the given index.
447 ///
448 /// Afterwards `self` contains elements `[0, at)`, and the returned `Bytes`
449 /// contains elements `[at, len)`. It's guaranteed that the memory does not
450 /// move, that is, the address of `self` does not change, and the address of
451 /// the returned slice is `at` bytes after that.
452 ///
453 /// This is an `O(1)` operation that just increases the reference count and
454 /// sets a few indices.
455 ///
456 /// # Examples
457 ///
458 /// ```
459 /// use bytes::Bytes;
460 ///
461 /// let mut a = Bytes::from(&b"hello world"[..]);
462 /// let b = a.split_off(5);
463 ///
464 /// assert_eq!(&a[..], b"hello");
465 /// assert_eq!(&b[..], b" world");
466 /// ```
467 ///
468 /// # Panics
469 ///
470 /// Panics if `at > len`.
471 #[must_use = "consider Bytes::truncate if you don't need the other half"]
472 pub fn split_off(&mut self, at: usize) -> Self {
473 if at == self.len() {
474 return Bytes::new_empty_with_ptr(self.ptr.wrapping_add(at));
475 }
476
477 if at == 0 {
478 return mem::replace(self, Bytes::new_empty_with_ptr(self.ptr));
479 }
480
481 assert!(
482 at <= self.len(),
483 "split_off out of bounds: {:?} <= {:?}",
484 at,
485 self.len(),
486 );
487
488 let mut ret = self.clone();
489
490 self.len = at;
491
492 unsafe { ret.inc_start(at) };
493
494 ret
495 }
496
497 /// Splits the bytes into two at the given index.
498 ///
499 /// Afterwards `self` contains elements `[at, len)`, and the returned
500 /// `Bytes` contains elements `[0, at)`.
501 ///
502 /// This is an `O(1)` operation that just increases the reference count and
503 /// sets a few indices.
504 ///
505 /// # Examples
506 ///
507 /// ```
508 /// use bytes::Bytes;
509 ///
510 /// let mut a = Bytes::from(&b"hello world"[..]);
511 /// let b = a.split_to(5);
512 ///
513 /// assert_eq!(&a[..], b" world");
514 /// assert_eq!(&b[..], b"hello");
515 /// ```
516 ///
517 /// # Panics
518 ///
519 /// Panics if `at > len`.
520 #[must_use = "consider Bytes::advance if you don't need the other half"]
521 pub fn split_to(&mut self, at: usize) -> Self {
522 if at == self.len() {
523 let end_ptr = self.ptr.wrapping_add(at);
524 return mem::replace(self, Bytes::new_empty_with_ptr(end_ptr));
525 }
526
527 if at == 0 {
528 return Bytes::new_empty_with_ptr(self.ptr);
529 }
530
531 assert!(
532 at <= self.len(),
533 "split_to out of bounds: {:?} <= {:?}",
534 at,
535 self.len(),
536 );
537
538 let mut ret = self.clone();
539
540 unsafe { self.inc_start(at) };
541
542 ret.len = at;
543 ret
544 }
545
546 /// Shortens the buffer, keeping the first `len` bytes and dropping the
547 /// rest.
548 ///
549 /// If `len` is greater than the buffer's current length, this has no
550 /// effect.
551 ///
552 /// The [split_off](`Self::split_off()`) method can emulate `truncate`, but this causes the
553 /// excess bytes to be returned instead of dropped.
554 ///
555 /// # Examples
556 ///
557 /// ```
558 /// use bytes::Bytes;
559 ///
560 /// let mut buf = Bytes::from(&b"hello world"[..]);
561 /// buf.truncate(5);
562 /// assert_eq!(buf, b"hello"[..]);
563 /// ```
564 #[inline]
565 pub fn truncate(&mut self, len: usize) {
566 if len < self.len {
567 // The Vec "promotable" vtables do not store the capacity,
568 // so we cannot truncate while using this repr. We *have* to
569 // promote using `split_off` so the capacity can be stored.
570 if self.vtable as *const Vtable == &PROMOTABLE_EVEN_VTABLE
571 || self.vtable as *const Vtable == &PROMOTABLE_ODD_VTABLE
572 {
573 drop(self.split_off(len));
574 } else {
575 self.len = len;
576 }
577 }
578 }
579
580 /// Clears the buffer, removing all data.
581 ///
582 /// # Examples
583 ///
584 /// ```
585 /// use bytes::Bytes;
586 ///
587 /// let mut buf = Bytes::from(&b"hello world"[..]);
588 /// buf.clear();
589 /// assert!(buf.is_empty());
590 /// ```
591 #[inline]
592 pub fn clear(&mut self) {
593 self.truncate(0);
594 }
595
596 /// Try to convert self into `BytesMut`.
597 ///
598 /// If `self` is unique for the entire original buffer, this will succeed
599 /// and return a `BytesMut` with the contents of `self` without copying.
600 /// If `self` is not unique for the entire original buffer, this will fail
601 /// and return self.
602 ///
603 /// This will also always fail if the buffer was constructed via either
604 /// [from_owner](Bytes::from_owner) or [from_static](Bytes::from_static).
605 ///
606 /// # Examples
607 ///
608 /// ```
609 /// use bytes::{Bytes, BytesMut};
610 ///
611 /// let bytes = Bytes::from(b"hello".to_vec());
612 /// assert_eq!(bytes.try_into_mut(), Ok(BytesMut::from(&b"hello"[..])));
613 /// ```
614 pub fn try_into_mut(self) -> Result<BytesMut, Bytes> {
615 if self.is_unique() {
616 Ok(self.into())
617 } else {
618 Err(self)
619 }
620 }
621
622 #[inline]
623 pub(crate) unsafe fn with_vtable(
624 ptr: *const u8,
625 len: usize,
626 data: AtomicPtr<()>,
627 vtable: &'static Vtable,
628 ) -> Bytes {
629 Bytes {
630 ptr,
631 len,
632 data,
633 vtable,
634 }
635 }
636
637 // private
638
639 #[inline]
640 fn as_slice(&self) -> &[u8] {
641 unsafe { slice::from_raw_parts(self.ptr, self.len) }
642 }
643
644 #[inline]
645 unsafe fn inc_start(&mut self, by: usize) {
646 // should already be asserted, but debug assert for tests
647 debug_assert!(self.len >= by, "internal: inc_start out of bounds");
648 self.len -= by;
649 self.ptr = self.ptr.add(by);
650 }
651
652 #[inline]
653 fn data_mut(&mut self) -> *mut () {
654 self.data.with_mut(|p| *p)
655 }
656}
657
658// Vtable must enforce this behavior
659unsafe impl Send for Bytes {}
660unsafe impl Sync for Bytes {}
661
662impl Drop for Bytes {
663 #[inline]
664 fn drop(&mut self) {
665 let data = self.data_mut();
666 unsafe { (self.vtable.drop)(data, self.ptr, self.len) }
667 }
668}
669
670impl Clone for Bytes {
671 #[inline]
672 fn clone(&self) -> Bytes {
673 unsafe { (self.vtable.clone)(&self.data, self.ptr, self.len) }
674 }
675}
676
677impl Buf for Bytes {
678 #[inline]
679 fn remaining(&self) -> usize {
680 self.len()
681 }
682
683 #[inline]
684 fn chunk(&self) -> &[u8] {
685 self.as_slice()
686 }
687
688 #[inline]
689 fn advance(&mut self, cnt: usize) {
690 assert!(
691 cnt <= self.len(),
692 "cannot advance past `remaining`: {:?} <= {:?}",
693 cnt,
694 self.len(),
695 );
696
697 unsafe {
698 self.inc_start(cnt);
699 }
700 }
701
702 fn copy_to_bytes(&mut self, len: usize) -> Self {
703 self.split_to(len)
704 }
705}
706
707impl Deref for Bytes {
708 type Target = [u8];
709
710 #[inline]
711 fn deref(&self) -> &[u8] {
712 self.as_slice()
713 }
714}
715
716impl AsRef<[u8]> for Bytes {
717 #[inline]
718 fn as_ref(&self) -> &[u8] {
719 self.as_slice()
720 }
721}
722
723impl hash::Hash for Bytes {
724 fn hash<H>(&self, state: &mut H)
725 where
726 H: hash::Hasher,
727 {
728 self.as_slice().hash(state);
729 }
730}
731
732impl Borrow<[u8]> for Bytes {
733 fn borrow(&self) -> &[u8] {
734 self.as_slice()
735 }
736}
737
738impl IntoIterator for Bytes {
739 type Item = u8;
740 type IntoIter = IntoIter<Bytes>;
741
742 fn into_iter(self) -> Self::IntoIter {
743 IntoIter::new(self)
744 }
745}
746
747impl<'a> IntoIterator for &'a Bytes {
748 type Item = &'a u8;
749 type IntoIter = core::slice::Iter<'a, u8>;
750
751 fn into_iter(self) -> Self::IntoIter {
752 self.as_slice().iter()
753 }
754}
755
756impl FromIterator<u8> for Bytes {
757 fn from_iter<T: IntoIterator<Item = u8>>(into_iter: T) -> Self {
758 Vec::from_iter(into_iter).into()
759 }
760}
761
762// impl Eq
763
764impl PartialEq for Bytes {
765 fn eq(&self, other: &Bytes) -> bool {
766 self.as_slice() == other.as_slice()
767 }
768}
769
770impl PartialOrd for Bytes {
771 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
772 Some(self.cmp(other))
773 }
774}
775
776impl Ord for Bytes {
777 fn cmp(&self, other: &Bytes) -> cmp::Ordering {
778 self.as_slice().cmp(other.as_slice())
779 }
780}
781
782impl Eq for Bytes {}
783
784impl PartialEq<[u8]> for Bytes {
785 fn eq(&self, other: &[u8]) -> bool {
786 self.as_slice() == other
787 }
788}
789
790impl PartialOrd<[u8]> for Bytes {
791 fn partial_cmp(&self, other: &[u8]) -> Option<cmp::Ordering> {
792 self.as_slice().partial_cmp(other)
793 }
794}
795
796impl PartialEq<Bytes> for [u8] {
797 fn eq(&self, other: &Bytes) -> bool {
798 *other == *self
799 }
800}
801
802impl PartialOrd<Bytes> for [u8] {
803 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
804 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
805 }
806}
807
808impl PartialEq<str> for Bytes {
809 fn eq(&self, other: &str) -> bool {
810 self.as_slice() == other.as_bytes()
811 }
812}
813
814impl PartialOrd<str> for Bytes {
815 fn partial_cmp(&self, other: &str) -> Option<cmp::Ordering> {
816 self.as_slice().partial_cmp(other.as_bytes())
817 }
818}
819
820impl PartialEq<Bytes> for str {
821 fn eq(&self, other: &Bytes) -> bool {
822 *other == *self
823 }
824}
825
826impl PartialOrd<Bytes> for str {
827 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
828 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
829 }
830}
831
832impl PartialEq<Vec<u8>> for Bytes {
833 fn eq(&self, other: &Vec<u8>) -> bool {
834 *self == other[..]
835 }
836}
837
838impl PartialOrd<Vec<u8>> for Bytes {
839 fn partial_cmp(&self, other: &Vec<u8>) -> Option<cmp::Ordering> {
840 self.as_slice().partial_cmp(&other[..])
841 }
842}
843
844impl PartialEq<Bytes> for Vec<u8> {
845 fn eq(&self, other: &Bytes) -> bool {
846 *other == *self
847 }
848}
849
850impl PartialOrd<Bytes> for Vec<u8> {
851 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
852 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
853 }
854}
855
856impl PartialEq<String> for Bytes {
857 fn eq(&self, other: &String) -> bool {
858 *self == other[..]
859 }
860}
861
862impl PartialOrd<String> for Bytes {
863 fn partial_cmp(&self, other: &String) -> Option<cmp::Ordering> {
864 self.as_slice().partial_cmp(other.as_bytes())
865 }
866}
867
868impl PartialEq<Bytes> for String {
869 fn eq(&self, other: &Bytes) -> bool {
870 *other == *self
871 }
872}
873
874impl PartialOrd<Bytes> for String {
875 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
876 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
877 }
878}
879
880impl PartialEq<Bytes> for &[u8] {
881 fn eq(&self, other: &Bytes) -> bool {
882 *other == *self
883 }
884}
885
886impl PartialOrd<Bytes> for &[u8] {
887 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
888 <[u8] as PartialOrd<[u8]>>::partial_cmp(self, other)
889 }
890}
891
892impl PartialEq<Bytes> for &str {
893 fn eq(&self, other: &Bytes) -> bool {
894 *other == *self
895 }
896}
897
898impl PartialOrd<Bytes> for &str {
899 fn partial_cmp(&self, other: &Bytes) -> Option<cmp::Ordering> {
900 <[u8] as PartialOrd<[u8]>>::partial_cmp(self.as_bytes(), other)
901 }
902}
903
904impl<'a, T: ?Sized> PartialEq<&'a T> for Bytes
905where
906 Bytes: PartialEq<T>,
907{
908 fn eq(&self, other: &&'a T) -> bool {
909 *self == **other
910 }
911}
912
913impl<'a, T: ?Sized> PartialOrd<&'a T> for Bytes
914where
915 Bytes: PartialOrd<T>,
916{
917 fn partial_cmp(&self, other: &&'a T) -> Option<cmp::Ordering> {
918 self.partial_cmp(&**other)
919 }
920}
921
922// impl From
923
924impl Default for Bytes {
925 #[inline]
926 fn default() -> Bytes {
927 Bytes::new()
928 }
929}
930
931impl From<&'static [u8]> for Bytes {
932 fn from(slice: &'static [u8]) -> Bytes {
933 Bytes::from_static(slice)
934 }
935}
936
937impl From<&'static str> for Bytes {
938 fn from(slice: &'static str) -> Bytes {
939 Bytes::from_static(slice.as_bytes())
940 }
941}
942
943impl From<Vec<u8>> for Bytes {
944 fn from(vec: Vec<u8>) -> Bytes {
945 let mut vec = ManuallyDrop::new(vec);
946 let ptr = vec.as_mut_ptr();
947 let len = vec.len();
948 let cap = vec.capacity();
949
950 // Avoid an extra allocation if possible.
951 if len == cap {
952 let vec = ManuallyDrop::into_inner(vec);
953 return Bytes::from(vec.into_boxed_slice());
954 }
955
956 let shared = Box::new(Shared {
957 buf: ptr,
958 cap,
959 ref_cnt: AtomicUsize::new(1),
960 });
961
962 let shared = Box::into_raw(shared);
963 // The pointer should be aligned, so this assert should
964 // always succeed.
965 debug_assert!(
966 0 == (shared as usize & KIND_MASK),
967 "internal: Box<Shared> should have an aligned pointer",
968 );
969 Bytes {
970 ptr,
971 len,
972 data: AtomicPtr::new(shared as _),
973 vtable: &SHARED_VTABLE,
974 }
975 }
976}
977
978impl From<Box<[u8]>> for Bytes {
979 fn from(slice: Box<[u8]>) -> Bytes {
980 // Box<[u8]> doesn't contain a heap allocation for empty slices,
981 // so the pointer isn't aligned enough for the KIND_VEC stashing to
982 // work.
983 if slice.is_empty() {
984 return Bytes::new();
985 }
986
987 let len = slice.len();
988 let ptr = Box::into_raw(slice) as *mut u8;
989
990 if ptr as usize & 0x1 == 0 {
991 let data = ptr_map(ptr, |addr| addr | KIND_VEC);
992 Bytes {
993 ptr,
994 len,
995 data: AtomicPtr::new(data.cast()),
996 vtable: &PROMOTABLE_EVEN_VTABLE,
997 }
998 } else {
999 Bytes {
1000 ptr,
1001 len,
1002 data: AtomicPtr::new(ptr.cast()),
1003 vtable: &PROMOTABLE_ODD_VTABLE,
1004 }
1005 }
1006 }
1007}
1008
1009impl From<Bytes> for BytesMut {
1010 /// Convert self into `BytesMut`.
1011 ///
1012 /// If `bytes` is unique for the entire original buffer, this will return a
1013 /// `BytesMut` with the contents of `bytes` without copying.
1014 /// If `bytes` is not unique for the entire original buffer, this will make
1015 /// a copy of `bytes` subset of the original buffer in a new `BytesMut`.
1016 ///
1017 /// # Examples
1018 ///
1019 /// ```
1020 /// use bytes::{Bytes, BytesMut};
1021 ///
1022 /// let bytes = Bytes::from(b"hello".to_vec());
1023 /// assert_eq!(BytesMut::from(bytes), BytesMut::from(&b"hello"[..]));
1024 /// ```
1025 fn from(bytes: Bytes) -> Self {
1026 let mut bytes = ManuallyDrop::new(bytes);
1027 let data = bytes.data_mut();
1028 unsafe { (bytes.vtable.into_mut)(data, bytes.ptr, bytes.len) }
1029 }
1030}
1031
1032impl From<String> for Bytes {
1033 fn from(s: String) -> Bytes {
1034 Bytes::from(s.into_bytes())
1035 }
1036}
1037
1038impl From<Bytes> for Vec<u8> {
1039 fn from(bytes: Bytes) -> Vec<u8> {
1040 let mut bytes = ManuallyDrop::new(bytes);
1041 let data = bytes.data_mut();
1042 unsafe { (bytes.vtable.into_vec)(data, bytes.ptr, bytes.len) }
1043 }
1044}
1045
1046// ===== impl Vtable =====
1047
1048impl fmt::Debug for Vtable {
1049 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1050 f.debug_struct("Vtable")
1051 .field("clone", &(self.clone as *const ()))
1052 .field("drop", &(self.drop as *const ()))
1053 .finish()
1054 }
1055}
1056
1057// ===== impl StaticVtable =====
1058
1059const STATIC_VTABLE: Vtable = Vtable {
1060 clone: static_clone,
1061 into_vec: static_to_vec,
1062 into_mut: static_to_mut,
1063 is_unique: static_is_unique,
1064 drop: static_drop,
1065};
1066
1067unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
1068 let slice = slice::from_raw_parts(ptr, len);
1069 Bytes::from_static(slice)
1070}
1071
1072unsafe fn static_to_vec(_: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
1073 let slice = slice::from_raw_parts(ptr, len);
1074 slice.to_vec()
1075}
1076
1077unsafe fn static_to_mut(_: *mut (), ptr: *const u8, len: usize) -> BytesMut {
1078 let slice = slice::from_raw_parts(ptr, len);
1079 BytesMut::from(slice)
1080}
1081
1082fn static_is_unique(_: &AtomicPtr<()>) -> bool {
1083 false
1084}
1085
1086unsafe fn static_drop(_: *mut (), _: *const u8, _: usize) {
1087 // nothing to drop for &'static [u8]
1088}
1089
1090// ===== impl OwnedVtable =====
1091
1092#[repr(C)]
1093struct Owned<T> {
1094 ref_cnt: AtomicUsize,
1095 owner: T,
1096}
1097
1098impl<T> Owned<T> {
1099 const VTABLE: Vtable = Vtable {
1100 clone: owned_clone::<T>,
1101 into_vec: owned_to_vec::<T>,
1102 into_mut: owned_to_mut::<T>,
1103 is_unique: owned_is_unique,
1104 drop: owned_drop::<T>,
1105 };
1106}
1107
1108unsafe fn owned_clone<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
1109 let owned = data.load(Ordering::Relaxed);
1110 let old_cnt = (*owned.cast::<AtomicUsize>()).fetch_add(1, Ordering::Relaxed);
1111 if old_cnt > usize::MAX >> 1 {
1112 crate::abort();
1113 }
1114
1115 Bytes {
1116 ptr,
1117 len,
1118 data: AtomicPtr::new(owned as _),
1119 vtable: &Owned::<T>::VTABLE,
1120 }
1121}
1122
1123unsafe fn owned_to_vec<T>(owned: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
1124 let slice = slice::from_raw_parts(ptr, len);
1125 let vec = slice.to_vec();
1126 owned_drop_impl::<T>(owned);
1127 vec
1128}
1129
1130unsafe fn owned_to_mut<T>(owned: *mut (), ptr: *const u8, len: usize) -> BytesMut {
1131 BytesMut::from_vec(owned_to_vec::<T>(owned, ptr, len))
1132}
1133
1134unsafe fn owned_is_unique(_data: &AtomicPtr<()>) -> bool {
1135 false
1136}
1137
1138unsafe fn owned_drop_impl<T>(owned: *mut ()) {
1139 {
1140 let ref_cnt = &*owned.cast::<AtomicUsize>();
1141
1142 let old_cnt = ref_cnt.fetch_sub(1, Ordering::Release);
1143 debug_assert!(
1144 old_cnt > 0 && old_cnt <= usize::MAX >> 1,
1145 "expected non-zero refcount and no underflow"
1146 );
1147 if old_cnt != 1 {
1148 return;
1149 }
1150 ref_cnt.load(Ordering::Acquire);
1151 }
1152
1153 drop(Box::<Owned<T>>::from_raw(owned.cast()));
1154}
1155
1156unsafe fn owned_drop<T>(data: *mut (), _ptr: *const u8, _len: usize) {
1157 owned_drop_impl::<T>(data);
1158}
1159
1160// ===== impl PromotableVtable =====
1161
1162static PROMOTABLE_EVEN_VTABLE: Vtable = Vtable {
1163 clone: promotable_even_clone,
1164 into_vec: promotable_even_to_vec,
1165 into_mut: promotable_even_to_mut,
1166 is_unique: promotable_is_unique,
1167 drop: promotable_even_drop,
1168};
1169
1170static PROMOTABLE_ODD_VTABLE: Vtable = Vtable {
1171 clone: promotable_odd_clone,
1172 into_vec: promotable_odd_to_vec,
1173 into_mut: promotable_odd_to_mut,
1174 is_unique: promotable_is_unique,
1175 drop: promotable_odd_drop,
1176};
1177
1178unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
1179 let shared = data.load(Ordering::Acquire);
1180 let kind = shared as usize & KIND_MASK;
1181
1182 if kind == KIND_ARC {
1183 shallow_clone_arc(shared.cast(), ptr, len)
1184 } else {
1185 debug_assert_eq!(kind, KIND_VEC);
1186 let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
1187 shallow_clone_vec(data, shared, buf, ptr, len)
1188 }
1189}
1190
1191unsafe fn promotable_to_vec(
1192 shared: *mut (),
1193 ptr: *const u8,
1194 len: usize,
1195 f: fn(*mut ()) -> *mut u8,
1196) -> Vec<u8> {
1197 let kind = shared as usize & KIND_MASK;
1198
1199 if kind == KIND_ARC {
1200 shared_to_vec_impl(shared.cast(), ptr, len)
1201 } else {
1202 // If Bytes holds a Vec, then the offset must be 0.
1203 debug_assert_eq!(kind, KIND_VEC);
1204
1205 let buf = f(shared);
1206
1207 let cap = ptr.offset_from(buf) as usize + len;
1208
1209 // Copy back buffer
1210 ptr::copy(ptr, buf, len);
1211
1212 Vec::from_raw_parts(buf, len, cap)
1213 }
1214}
1215
1216unsafe fn promotable_to_mut(
1217 shared: *mut (),
1218 ptr: *const u8,
1219 len: usize,
1220 f: fn(*mut ()) -> *mut u8,
1221) -> BytesMut {
1222 let kind = shared as usize & KIND_MASK;
1223
1224 if kind == KIND_ARC {
1225 shared_to_mut_impl(shared.cast(), ptr, len)
1226 } else {
1227 // KIND_VEC is a view of an underlying buffer at a certain offset.
1228 // The ptr + len always represents the end of that buffer.
1229 // Before truncating it, it is first promoted to KIND_ARC.
1230 // Thus, we can safely reconstruct a Vec from it without leaking memory.
1231 debug_assert_eq!(kind, KIND_VEC);
1232
1233 let buf = f(shared);
1234 let off = ptr.offset_from(buf) as usize;
1235 let cap = off + len;
1236 let v = Vec::from_raw_parts(buf, cap, cap);
1237
1238 let mut b = BytesMut::from_vec(v);
1239 b.advance_unchecked(off);
1240 b
1241 }
1242}
1243
1244unsafe fn promotable_even_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
1245 promotable_to_vec(shared, ptr, len, |shared| {
1246 ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
1247 })
1248}
1249
1250unsafe fn promotable_even_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
1251 promotable_to_mut(shared, ptr, len, |shared| {
1252 ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
1253 })
1254}
1255
1256unsafe fn promotable_even_drop(shared: *mut (), ptr: *const u8, len: usize) {
1257 let kind = shared as usize & KIND_MASK;
1258
1259 if kind == KIND_ARC {
1260 release_shared(shared.cast());
1261 } else {
1262 debug_assert_eq!(kind, KIND_VEC);
1263 let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
1264 free_boxed_slice(buf, ptr, len);
1265 }
1266}
1267
1268unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
1269 let shared = data.load(Ordering::Acquire);
1270 let kind = shared as usize & KIND_MASK;
1271
1272 if kind == KIND_ARC {
1273 shallow_clone_arc(shared as _, ptr, len)
1274 } else {
1275 debug_assert_eq!(kind, KIND_VEC);
1276 shallow_clone_vec(data, shared, shared.cast(), ptr, len)
1277 }
1278}
1279
1280unsafe fn promotable_odd_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
1281 promotable_to_vec(shared, ptr, len, |shared| shared.cast())
1282}
1283
1284unsafe fn promotable_odd_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
1285 promotable_to_mut(shared, ptr, len, |shared| shared.cast())
1286}
1287
1288unsafe fn promotable_odd_drop(shared: *mut (), ptr: *const u8, len: usize) {
1289 let kind = shared as usize & KIND_MASK;
1290
1291 if kind == KIND_ARC {
1292 release_shared(shared.cast());
1293 } else {
1294 debug_assert_eq!(kind, KIND_VEC);
1295
1296 free_boxed_slice(shared.cast(), ptr, len);
1297 }
1298}
1299
1300unsafe fn promotable_is_unique(data: &AtomicPtr<()>) -> bool {
1301 let shared = data.load(Ordering::Acquire);
1302 let kind = shared as usize & KIND_MASK;
1303
1304 if kind == KIND_ARC {
1305 let ref_cnt = (*shared.cast::<Shared>()).ref_cnt.load(Ordering::Relaxed);
1306 ref_cnt == 1
1307 } else {
1308 true
1309 }
1310}
1311
1312unsafe fn free_boxed_slice(buf: *mut u8, offset: *const u8, len: usize) {
1313 let cap = offset.offset_from(buf) as usize + len;
1314 dealloc(buf, Layout::from_size_align(cap, 1).unwrap())
1315}
1316
1317// ===== impl SharedVtable =====
1318
1319struct Shared {
1320 // Holds arguments to dealloc upon Drop, but otherwise doesn't use them
1321 buf: *mut u8,
1322 cap: usize,
1323 ref_cnt: AtomicUsize,
1324}
1325
1326impl Drop for Shared {
1327 fn drop(&mut self) {
1328 unsafe { dealloc(self.buf, Layout::from_size_align(self.cap, 1).unwrap()) }
1329 }
1330}
1331
1332// Assert that the alignment of `Shared` is divisible by 2.
1333// This is a necessary invariant since we depend on allocating `Shared` a
1334// shared object to implicitly carry the `KIND_ARC` flag in its pointer.
1335// This flag is set when the LSB is 0.
1336const _: [(); 0 - mem::align_of::<Shared>() % 2] = []; // Assert that the alignment of `Shared` is divisible by 2.
1337
1338static SHARED_VTABLE: Vtable = Vtable {
1339 clone: shared_clone,
1340 into_vec: shared_to_vec,
1341 into_mut: shared_to_mut,
1342 is_unique: shared_is_unique,
1343 drop: shared_drop,
1344};
1345
1346const KIND_ARC: usize = 0b0;
1347const KIND_VEC: usize = 0b1;
1348const KIND_MASK: usize = 0b1;
1349
1350unsafe fn shared_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
1351 let shared = data.load(Ordering::Relaxed);
1352 shallow_clone_arc(shared as _, ptr, len)
1353}
1354
1355unsafe fn shared_to_vec_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> Vec<u8> {
1356 // Check that the ref_cnt is 1 (unique).
1357 //
1358 // If it is unique, then it is set to 0 with AcqRel fence for the same
1359 // reason in release_shared.
1360 //
1361 // Otherwise, we take the other branch and call release_shared.
1362 if (*shared)
1363 .ref_cnt
1364 .compare_exchange(1, 0, Ordering::AcqRel, Ordering::Relaxed)
1365 .is_ok()
1366 {
1367 // Deallocate the `Shared` instance without running its destructor.
1368 let shared = *Box::from_raw(shared);
1369 let shared = ManuallyDrop::new(shared);
1370 let buf = shared.buf;
1371 let cap = shared.cap;
1372
1373 // Copy back buffer
1374 ptr::copy(ptr, buf, len);
1375
1376 Vec::from_raw_parts(buf, len, cap)
1377 } else {
1378 let v = slice::from_raw_parts(ptr, len).to_vec();
1379 release_shared(shared);
1380 v
1381 }
1382}
1383
1384unsafe fn shared_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
1385 shared_to_vec_impl(shared.cast(), ptr, len)
1386}
1387
1388unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> BytesMut {
1389 // The goal is to check if the current handle is the only handle
1390 // that currently has access to the buffer. This is done by
1391 // checking if the `ref_cnt` is currently 1.
1392 //
1393 // The `Acquire` ordering synchronizes with the `Release` as
1394 // part of the `fetch_sub` in `release_shared`. The `fetch_sub`
1395 // operation guarantees that any mutations done in other threads
1396 // are ordered before the `ref_cnt` is decremented. As such,
1397 // this `Acquire` will guarantee that those mutations are
1398 // visible to the current thread.
1399 //
1400 // Otherwise, we take the other branch, copy the data and call `release_shared`.
1401 if (*shared).ref_cnt.load(Ordering::Acquire) == 1 {
1402 // Deallocate the `Shared` instance without running its destructor.
1403 let shared = *Box::from_raw(shared);
1404 let shared = ManuallyDrop::new(shared);
1405 let buf = shared.buf;
1406 let cap = shared.cap;
1407
1408 // Rebuild Vec
1409 let off = ptr.offset_from(buf) as usize;
1410 let v = Vec::from_raw_parts(buf, len + off, cap);
1411
1412 let mut b = BytesMut::from_vec(v);
1413 b.advance_unchecked(off);
1414 b
1415 } else {
1416 // Copy the data from Shared in a new Vec, then release it
1417 let v = slice::from_raw_parts(ptr, len).to_vec();
1418 release_shared(shared);
1419 BytesMut::from_vec(v)
1420 }
1421}
1422
1423unsafe fn shared_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
1424 shared_to_mut_impl(shared.cast(), ptr, len)
1425}
1426
1427pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool {
1428 let shared = data.load(Ordering::Acquire);
1429 let ref_cnt = (*shared.cast::<Shared>()).ref_cnt.load(Ordering::Relaxed);
1430 ref_cnt == 1
1431}
1432
1433unsafe fn shared_drop(shared: *mut (), _ptr: *const u8, _len: usize) {
1434 release_shared(shared.cast());
1435}
1436
1437unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes {
1438 let old_size = (*shared).ref_cnt.fetch_add(1, Ordering::Relaxed);
1439
1440 if old_size > usize::MAX >> 1 {
1441 crate::abort();
1442 }
1443
1444 Bytes {
1445 ptr,
1446 len,
1447 data: AtomicPtr::new(shared as _),
1448 vtable: &SHARED_VTABLE,
1449 }
1450}
1451
1452#[cold]
1453unsafe fn shallow_clone_vec(
1454 atom: &AtomicPtr<()>,
1455 ptr: *const (),
1456 buf: *mut u8,
1457 offset: *const u8,
1458 len: usize,
1459) -> Bytes {
1460 // If the buffer is still tracked in a `Vec<u8>`. It is time to
1461 // promote the vec to an `Arc`. This could potentially be called
1462 // concurrently, so some care must be taken.
1463
1464 // First, allocate a new `Shared` instance containing the
1465 // `Vec` fields. It's important to note that `ptr`, `len`,
1466 // and `cap` cannot be mutated without having `&mut self`.
1467 // This means that these fields will not be concurrently
1468 // updated and since the buffer hasn't been promoted to an
1469 // `Arc`, those three fields still are the components of the
1470 // vector.
1471 let shared = Box::new(Shared {
1472 buf,
1473 cap: offset.offset_from(buf) as usize + len,
1474 // Initialize refcount to 2. One for this reference, and one
1475 // for the new clone that will be returned from
1476 // `shallow_clone`.
1477 ref_cnt: AtomicUsize::new(2),
1478 });
1479
1480 let shared = Box::into_raw(shared);
1481
1482 // The pointer should be aligned, so this assert should
1483 // always succeed.
1484 debug_assert!(
1485 0 == (shared as usize & KIND_MASK),
1486 "internal: Box<Shared> should have an aligned pointer",
1487 );
1488
1489 // Try compare & swapping the pointer into the `arc` field.
1490 // `Release` is used synchronize with other threads that
1491 // will load the `arc` field.
1492 //
1493 // If the `compare_exchange` fails, then the thread lost the
1494 // race to promote the buffer to shared. The `Acquire`
1495 // ordering will synchronize with the `compare_exchange`
1496 // that happened in the other thread and the `Shared`
1497 // pointed to by `actual` will be visible.
1498 match atom.compare_exchange(ptr as _, shared as _, Ordering::AcqRel, Ordering::Acquire) {
1499 Ok(actual) => {
1500 debug_assert!(core::ptr::eq(actual, ptr));
1501 // The upgrade was successful, the new handle can be
1502 // returned.
1503 Bytes {
1504 ptr: offset,
1505 len,
1506 data: AtomicPtr::new(shared as _),
1507 vtable: &SHARED_VTABLE,
1508 }
1509 }
1510 Err(actual) => {
1511 // The upgrade failed, a concurrent clone happened. Release
1512 // the allocation that was made in this thread, it will not
1513 // be needed.
1514 let shared = Box::from_raw(shared);
1515 mem::forget(*shared);
1516
1517 // Buffer already promoted to shared storage, so increment ref
1518 // count.
1519 shallow_clone_arc(actual as _, offset, len)
1520 }
1521 }
1522}
1523
1524unsafe fn release_shared(ptr: *mut Shared) {
1525 // `Shared` storage... follow the drop steps from Arc.
1526 if (*ptr).ref_cnt.fetch_sub(1, Ordering::Release) != 1 {
1527 return;
1528 }
1529
1530 // This fence is needed to prevent reordering of use of the data and
1531 // deletion of the data. Because it is marked `Release`, the decreasing
1532 // of the reference count synchronizes with this `Acquire` fence. This
1533 // means that use of the data happens before decreasing the reference
1534 // count, which happens before this fence, which happens before the
1535 // deletion of the data.
1536 //
1537 // As explained in the [Boost documentation][1],
1538 //
1539 // > It is important to enforce any possible access to the object in one
1540 // > thread (through an existing reference) to *happen before* deleting
1541 // > the object in a different thread. This is achieved by a "release"
1542 // > operation after dropping a reference (any access to the object
1543 // > through this reference must obviously happened before), and an
1544 // > "acquire" operation before deleting the object.
1545 //
1546 // [1]: (www.boost.org/doc/libs/1_55_0/doc/html/atomic/usage_examples.html)
1547 //
1548 // Thread sanitizer does not support atomic fences. Use an atomic load
1549 // instead.
1550 (*ptr).ref_cnt.load(Ordering::Acquire);
1551
1552 // Drop the data
1553 drop(Box::from_raw(ptr));
1554}
1555
1556// Ideally we would always use this version of `ptr_map` since it is strict
1557// provenance compatible, but it results in worse codegen. We will however still
1558// use it on miri because it gives better diagnostics for people who test bytes
1559// code with miri.
1560//
1561// See https://github.com/tokio-rs/bytes/pull/545 for more info.
1562#[cfg(miri)]
1563fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8
1564where
1565 F: FnOnce(usize) -> usize,
1566{
1567 let old_addr = ptr as usize;
1568 let new_addr = f(old_addr);
1569 let diff = new_addr.wrapping_sub(old_addr);
1570 ptr.wrapping_add(diff)
1571}
1572
1573#[cfg(not(miri))]
1574fn ptr_map<F>(ptr: *mut u8, f: F) -> *mut u8
1575where
1576 F: FnOnce(usize) -> usize,
1577{
1578 let old_addr = ptr as usize;
1579 let new_addr = f(old_addr);
1580 new_addr as *mut u8
1581}
1582
1583fn without_provenance(ptr: usize) -> *const u8 {
1584 core::ptr::null::<u8>().wrapping_add(ptr)
1585}
1586
1587// compile-fails
1588
1589/// ```compile_fail
1590/// use bytes::Bytes;
1591/// #[deny(unused_must_use)]
1592/// {
1593/// let mut b1 = Bytes::from("hello world");
1594/// b1.split_to(6);
1595/// }
1596/// ```
1597fn _split_to_must_use() {}
1598
1599/// ```compile_fail
1600/// use bytes::Bytes;
1601/// #[deny(unused_must_use)]
1602/// {
1603/// let mut b1 = Bytes::from("hello world");
1604/// b1.split_off(6);
1605/// }
1606/// ```
1607fn _split_off_must_use() {}
1608
1609// fuzz tests
1610#[cfg(all(test, loom))]
1611mod fuzz {
1612 use loom::sync::Arc;
1613 use loom::thread;
1614
1615 use super::Bytes;
1616 #[test]
1617 fn bytes_cloning_vec() {
1618 loom::model(|| {
1619 let a = Bytes::from(b"abcdefgh".to_vec());
1620 let addr = a.as_ptr() as usize;
1621
1622 // test the Bytes::clone is Sync by putting it in an Arc
1623 let a1 = Arc::new(a);
1624 let a2 = a1.clone();
1625
1626 let t1 = thread::spawn(move || {
1627 let b: Bytes = (*a1).clone();
1628 assert_eq!(b.as_ptr() as usize, addr);
1629 });
1630
1631 let t2 = thread::spawn(move || {
1632 let b: Bytes = (*a2).clone();
1633 assert_eq!(b.as_ptr() as usize, addr);
1634 });
1635
1636 t1.join().unwrap();
1637 t2.join().unwrap();
1638 });
1639 }
1640}