oxedyne/fe2o3/fe2o3_o3db_sync/tests/cdc_cut_stream.rs
1.2 KiB, 1 run
created by r1053032297:77832, which is this file's identity for as long as the history lasts, whatever it is later renamed to
download · who wrote it · its history
| 1 | //! A stream cut window by window must give the chunks that `split` gives over the whole. |
| 2 | //! |
| 3 | //! A caller that cannot hold a file in memory reads a window of `max_size()` bytes, takes the |
| 4 | //! first cut and moves on. If a cut depended on bytes past the window, the two would disagree |
| 5 | //! and the same file would chunk differently on two devices. |
| 6 | |
| 7 | use oxedyne_fe2o3_core::prelude::*; |
| 8 | use oxedyne_fe2o3_o3db_sync::cas::CdcChunker; |
| 9 | |
| 10 | #[test] |
| 11 | fn a_stream_cut_by_windows_matches_split() -> Outcome<()> { |
| 12 | let cdc = res!(CdcChunker::new(64, 256, 1024)); |
| 13 | let mut x = 0x9e3779b97f4a7c15u64; // fixed: a seed, any non-zero constant serves. |
| 14 | let data: Vec<u8> = (0..100_000).map(|_| { |
| 15 | x ^= x << 13; |
| 16 | x ^= x >> 7; |
| 17 | x ^= x << 17; |
| 18 | (x >> 24) as u8 |
| 19 | }).collect(); |
| 20 | let (manifest, _) = cdc.split(&data); |
| 21 | let want: Vec<usize> = manifest.chunks.iter().map(|c| c.len).collect(); |
| 22 | let mut lens = Vec::new(); |
| 23 | let mut pos = 0; |
| 24 | while pos < data.len() { |
| 25 | let end = (pos + cdc.max_size()).min(data.len()); // The window a stream reader holds. |
| 26 | let len = cdc.cut(&data[pos..end]); |
| 27 | lens.push(len); |
| 28 | pos += len; |
| 29 | } |
| 30 | assert!(want.len() > 100, "The data must cut into many chunks, got {}.", want.len()); |
| 31 | assert_eq!(lens, want); |
| 32 | Ok(()) |
| 33 | } |