Oregami
Repositories/oxedyne/fe2o3

oxedyne/fe2o3/fe2o3_o3db_sync/src/file/stored.rs

9.0 KiB, 30 runs

created by r1870400018:801, which is this file's identity for as long as the history lasts, whatever it is later renamed to

download · who wrote it · its history

1use crate::{
2 prelude::*,
3 data::{
4 core::Key,
5 },
6 file::floc::{
7 FileLocation,
8 FileNum,
9 StoredFileLocation,
10 },
11};
12
13use oxedyne_fe2o3_core::{
14 byte::ToBytes,
15};
16use oxedyne_fe2o3_jdat::{
17 prelude::*,
18 id::NumIdDat,
19};
20use oxedyne_fe2o3_hash::csum::ChecksumScheme;
21use oxedyne_fe2o3_iop_db::api::Meta;
22use oxedyne_fe2o3_iop_hash::csum::Checksummer;
23
24use std::{
25 io::{
26 self,
27 SeekFrom,
28 },
29};
30
31/// Names one record, by a digest of its key and the stamp it was written with. An offset names a
32/// record only within one generation of its file: a collection moves the records it keeps, and
33/// with records of one size a new offset can equal an old one still in use elsewhere, so a
34/// location matched by offset alone can be taken for another record (2026-09-23).
35#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
36pub struct RecordDigest(u64);
37
38impl RecordDigest {
39
40 pub fn new<
41 const UIDL: usize,
42 UID: NumIdDat<UIDL>,
43 >(
44 key: &[u8],
45 meta: &Meta<UIDL, UID>,
46 )
47 -> Outcome<Self>
48 {
49 // The key bytes are a daticle, which ends itself, so the stamp cannot run into it.
50 let mut buf = Vec::with_capacity(key.len() + Meta::<UIDL, UID>::BYTE_LEN);
51 buf.extend_from_slice(key);
52 buf = res!(meta.to_bytes(buf));
53 Ok(Self(seahash::hash(&buf)))
54 }
55}
56
57#[derive(Debug)]
58pub struct StoredKey<
59 const UIDL: usize,
60 UID: NumIdDat<UIDL>,
61> {
62 chash: alias::ChooseHash, // Hash used to determine cache pathway.
63 key: Key,
64 meta: Meta<UIDL, UID>,
65 csum: Vec<u8>,
66}
67
68impl<
69 const UIDL: usize,
70 UID: NumIdDat<UIDL>,
71>
72 StoredKey<UIDL, UID>
73{
74 pub fn ref_chash(&self) -> &alias::ChooseHash { &self.chash }
75 pub fn mut_chash(&mut self) -> &mut alias::ChooseHash { &mut self.chash }
76 pub fn key(&self) -> &Key { &self.key }
77 pub fn into_key(self) -> Key { self.key }
78 pub fn meta(&self) -> &Meta<UIDL, UID> { &self.meta }
79
80 /// This associated function tries to avoid unnecessary copying during the encoding of a
81 /// `StoredKey` by accepting a mutable reference to a vector of key bytes and appending a
82 /// timestamp and a checksum.
83 pub fn build_bytes<C: Checksummer>(
84 chash: alias::ChooseHash,
85 mut buf: Vec<u8>,
86 cind: Option<usize>,
87 meta: &Meta<UIDL, UID>,
88 csummer: C,
89 )
90 -> Outcome<Vec<u8>>
91 {
92 let mut cbuf = chash.to_vec();
93 cbuf.append(&mut buf);
94 buf = cbuf;
95 // 0. Append chunk index information.
96 let cind = match cind {
97 Some(uint) => Some(try_into!(u64, uint)),
98 None => None::<u64>,
99 };
100 let dat_cind = dat!(cind);
101 buf = res!(dat_cind.to_bytes(buf));
102 // 1. Append meta to key.
103 buf = res!(meta.to_bytes(buf));
104 // 2. Calculate checksum and append.
105 (buf, _) = res!(csummer.append(buf));
106 Ok(buf)
107 }
108
109 /// Do these stored key bytes, as they lie in a data file, hold this key with this stamp?
110 /// Read in place, since this is on every read from a file: the key follows the cache hash,
111 /// and the stamp comes just before the checksum. Key bytes are a daticle, which ends itself,
112 /// so equal bytes at the head of the stored key are the whole of its key.
113 pub fn holds(
114 stored: &[u8],
115 key: &[u8],
116 meta: &Meta<UIDL, UID>,
117 csum_len: usize,
118 )
119 -> Outcome<bool>
120 {
121 let key_end = constant::CACHE_HASH_BYTES + key.len();
122 let meta_end = match stored.len().checked_sub(csum_len) {
123 Some(n) => n,
124 None => return Ok(false),
125 };
126 let meta_start = match meta_end.checked_sub(Meta::<UIDL, UID>::BYTE_LEN) {
127 Some(n) => n,
128 None => return Ok(false),
129 };
130 if key_end > meta_start || &stored[constant::CACHE_HASH_BYTES..key_end] != key {
131 return Ok(false);
132 }
133 let (stamp, _) = res!(Meta::<UIDL, UID>::from_bytes(&stored[meta_start..meta_end]));
134 Ok(stamp == *meta)
135 }
136
137 /// Returns the key bytes, the complete `StoredKey` bytes and the length.
138 pub fn load<
139 C: Checksummer,
140 R: io::Read,
141 >(
142 mut r: &mut R,
143 csummer: C,
144 )
145 -> Outcome<Option<(Self, Vec<u8>, usize)>>
146 {
147 // Load the chash bytes.
148 let mut chash = alias::ChooseHash::default();
149 match r.read_exact(&mut chash) {
150 Err(e) => match e.kind() {
151 std::io::ErrorKind::UnexpectedEof => return Ok(None),
152 _ => return Err(err!(e,
153 "While trying to read cache hash from the start of a new key.";
154 Decode, Bytes)),
155 }
156 _ => (),
157 }
158 let mut skey = chash.to_vec();
159
160 // Load key Daticle.
161 let keybyts = res!(Dat::load_bytes(&mut r), Decode, Bytes);
162 skey.extend_from_slice(&keybyts[..]);
163
164 // Load chunk index information.
165 let cind_byts = res!(Dat::load_bytes(&mut r), Decode, Bytes);
166 let (dat_cind, _) = res!(Dat::from_bytes(&cind_byts));
167 let cind = match &dat_cind {
168 Dat::Opt(boxoptd) => {
169 match **boxoptd {
170 Some(Dat::U64(i)) => Some(try_into!(usize, i)),
171 None => None,
172 _ => return Err(err!(
173 "Expected Dat::Opt(Dat::U64), decoded {:?}.", dat_cind;
174 Invalid, Input)),
175 }
176 },
177 _ => {
178 //debug!(sync_log::stream(), "chash={:02x?} keybyts={:02x?} cind_byts={:02x?}",chash,keybyts,cind_byts);
179 return Err(err!(
180 "Expected Dat::Opt(Dat::U64), decoded {:?}.", dat_cind;
181 Invalid, Input));
182 },
183 };
184 skey.extend_from_slice(&cind_byts);
185
186 // Load rest of data in one go, but we're forced to use a vec because of the potentially
187 // variable checksummer byte length.
188 let mut buf = vec![0; Meta::<UIDL, UID>::BYTE_LEN + res!(csummer.len())];
189 res!(r.read_exact(&mut buf), Decode, Bytes);
190
191 // Fuse with the key.
192 skey.extend_from_slice(&buf);
193 let skey_len = skey.len();
194
195 // Verify checksum.
196 let csum = res!(csummer.verify(&skey));
197
198 // Remove the cache hash from the front of the key, now that the overall checksum has been
199 // verified.
200 skey.drain(..constant::CACHE_HASH_BYTES);
201
202 // Read Meta data.
203 let (meta, _) = res!(Meta::from_bytes(&buf));
204
205 Ok(Some((
206 Self {
207 chash,
208 key: match cind {
209 Some(i) => Key::Chunk(keybyts, i),
210 None => Key::Complete(keybyts),
211 },
212 meta,
213 csum,
214 },
215 skey,
216 skey_len,
217 )))
218 }
219
220}
221
222pub struct StoredValue {}
223
224impl StoredValue {
225
226 /// This associated function is intended to avoid unnecessary copying during the encoding of a
227 /// `StoredValue` by accepting a mutable reference to a vector of value bytes and appending a
228 /// checksum.
229 pub fn build_bytes<C: Checksummer>(
230 mut buf: Vec<u8>,
231 csummer: C,
232 )
233 -> Outcome<Vec<u8>>
234 {
235 (buf, _) = res!(csummer.append(buf));
236 Ok(buf)
237 }
238
239 pub fn count<RS: io::Read + io::Seek>(
240 mut rs: &mut RS,
241 csumlen: usize,
242 )
243 -> Outcome<usize>
244 {
245 // 1. Count value Daticle bytes.
246 let n = res!(Dat::count_bytes(&mut rs), Decode, Bytes);
247 if n == 0 {
248 return Ok(0);
249 }
250 let csumlen_i64 = try_into!(i64, csumlen);
251 res!(rs.seek(SeekFrom::Current(csumlen_i64)));
252 Ok(n + csumlen)
253 }
254
255}
256
257#[derive(Debug)]
258pub struct StoredIndex {
259 sfloc: StoredFileLocation,
260 csum: Vec<u8>,
261}
262
263impl StoredIndex {
264
265 pub fn keyval_len(&self) -> u64 {
266 self.sfloc.floc.keyval().len
267 }
268
269 pub fn ref_file_location(&self) -> &FileLocation { &self.sfloc.floc }
270 pub fn own_file_location(self) -> FileLocation { self.sfloc.floc }
271 pub fn ref_stored_file_location(&self) -> &StoredFileLocation { &self.sfloc }
272 pub fn own_stored_file_location(self) -> StoredFileLocation { self.sfloc }
273
274 pub fn read<
275 C: Checksummer,
276 R: io::Read,
277 >(
278 r: &mut R,
279 fnum: FileNum,
280 csummer: C,
281 )
282 -> Outcome<(Option<Self>, usize)>
283 {
284 // Read a StoredFileLocation, minus the checksum (but calculating a checksum).
285 let (sfloc, n, csum1) = res!(StoredFileLocation::read_bytes(r, fnum, csummer.clone()));
286 let (csum2, n2) = res!(csummer.read_bytes(r));
287 res!(ChecksumScheme::compare(&csum1, &csum2));
288 Ok((
289 Some(Self {
290 sfloc: sfloc,
291 csum: csum2,
292 }),
293 n + n2,
294 ))
295 }
296
297}