+6 -2

knot2/third_party/gix-pack/src/bundle/find.rs

View file Reviewed

··· 14 14 out: &'a mut Vec<u8>, 15 15 inflate: &mut zlib::Inflate, 16 16 cache: &mut dyn crate::cache::DecodeEntry, 17 17 - ) -> Result<Option<(gix_object::Data<'a>, crate::data::entry::Location)>, crate::data::decode::Error> { 17 17 + ) -> Result< 18 18 + Option<(gix_object::Data<'a>, crate::data::entry::Location)>, 19 19 + crate::data::decode::Error, 20 20 + > { 18 21 let idx = match self.index.lookup(id) { 19 22 Some(idx) => idx, 20 23 None => return Ok(None), ··· 35 38 out: &'a mut Vec<u8>, 36 39 inflate: &mut zlib::Inflate, 37 40 cache: &mut dyn crate::cache::DecodeEntry, 38 38 - ) -> Result<(gix_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> { 41 41 + ) -> Result<(gix_object::Data<'a>, crate::data::entry::Location), crate::data::decode::Error> 42 42 + { 39 43 let ofs = self.index.pack_offset_at_index(idx); 40 44 let pack_entry = self.pack.entry(ofs)?; 41 45 let header_size = pack_entry.header_size();

+4 -1

knot2/third_party/gix-pack/src/bundle/mod.rs

View file Reviewed

··· 33 33 progress: &mut dyn DynNestedProgress, 34 34 should_interrupt: &AtomicBool, 35 35 options: crate::index::verify::integrity::Options<F>, 36 36 - ) -> Result<integrity::Outcome, crate::index::traverse::Error<crate::index::verify::integrity::Error>> 36 36 + ) -> Result< 37 37 + integrity::Outcome, 38 38 + crate::index::traverse::Error<crate::index::verify::integrity::Error>, 39 39 + > 37 40 where 38 41 C: crate::cache::DecodeEntry, 39 42 F: Fn() -> C + Send + Clone,

+45 -14

knot2/third_party/gix-pack/src/bundle/write/mod.rs

View file Reviewed

··· 68 68 options: Options, 69 69 ) -> Result<Outcome, Error> { 70 70 let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory()"); 71 71 - let mut read_progress = progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); 71 71 + let mut read_progress = 72 72 + progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); 72 73 read_progress.init(None, progress::bytes()); 73 74 let pack = progress::Read { 74 75 inner: pack, ··· 79 80 let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::with_capacity( 80 81 64 * 1024, 81 82 match directory.as_ref() { 82 82 - Some(directory) => gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?, 83 83 - None => gix_tempfile::new(std::env::temp_dir(), ContainingDirectory::Exists, AutoRemove::Tempfile)?, 83 83 + Some(directory) => { 84 84 + gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)? 85 85 + } 86 86 + None => gix_tempfile::new( 87 87 + std::env::temp_dir(), 88 88 + ContainingDirectory::Exists, 89 89 + AutoRemove::Tempfile, 90 90 + )?, 84 91 }, 85 92 ))); 86 93 let (pack_entries_iter, pack_version): ( ··· 178 185 options: Options, 179 186 ) -> Result<Outcome, Error> { 180 187 let _span = gix_features::trace::coarse!("gix_pack::Bundle::write_to_directory_eagerly()"); 181 181 - let mut read_progress = progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); /* Bundle Write Read pack Bytes*/ 188 188 + let mut read_progress = 189 189 + progress.add_child_with_id("read pack".into(), ProgressId::ReadPackBytes.into()); /* Bundle Write Read pack Bytes*/ 182 190 read_progress.init(pack_size.map(|s| s as usize), progress::bytes()); 183 191 let pack = progress::Read { 184 192 inner: pack, 185 193 progress: progress::ThroughputOnDrop::new(read_progress), 186 194 }; 187 195 188 188 - let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::new(match directory.as_ref() { 189 189 - Some(directory) => gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?, 190 190 - None => gix_tempfile::new(std::env::temp_dir(), ContainingDirectory::Exists, AutoRemove::Tempfile)?, 191 191 - }))); 196 196 + let data_file = Arc::new(parking_lot::Mutex::new(io::BufWriter::new( 197 197 + match directory.as_ref() { 198 198 + Some(directory) => { 199 199 + gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)? 200 200 + } 201 201 + None => gix_tempfile::new( 202 202 + std::env::temp_dir(), 203 203 + ContainingDirectory::Exists, 204 204 + AutoRemove::Tempfile, 205 205 + )?, 206 206 + }, 207 207 + ))); 192 208 let object_hash = options.object_hash; 193 209 let eight_pages = 4096 * 8; 194 210 let (pack_entries_iter, pack_version): ( 195 195 - Box<dyn Iterator<Item = Result<data::input::Entry, data::input::Error>> + Send + 'static>, 211 211 + Box< 212 212 + dyn Iterator<Item = Result<data::input::Entry, data::input::Error>> 213 213 + + Send 214 214 + + 'static, 215 215 + >, 196 216 _, 197 217 ) = match thin_pack_base_object_lookup { 198 218 Some(thin_pack_lookup) => { ··· 233 253 } 234 254 }; 235 255 let num_objects = pack_entries_iter.size_hint().0; 236 236 - let pack_entries_iter = 237 237 - gix_features::parallel::EagerIterIf::new(move || num_objects > 25_000, pack_entries_iter, 5_000, 5); 256 256 + let pack_entries_iter = gix_features::parallel::EagerIterIf::new( 257 257 + move || num_objects > 25_000, 258 258 + pack_entries_iter, 259 259 + 5_000, 260 260 + 5, 261 261 + ); 238 262 239 263 let WriteOutcome { 240 264 outcome, ··· 271 295 object_hash, 272 296 }: Options, 273 297 data_file: SharedTempFile, 274 274 - mut pack_entries_iter: Box<dyn Iterator<Item = Result<data::input::Entry, data::input::Error>> + 'a>, 298 298 + mut pack_entries_iter: Box< 299 299 + dyn Iterator<Item = Result<data::input::Entry, data::input::Error>> + 'a, 300 300 + >, 275 301 should_interrupt: &AtomicBool, 276 302 pack_version: data::Version, 277 303 ) -> Result<WriteOutcome, Error> { ··· 282 308 Ok(match directory { 283 309 Some(directory) => { 284 310 let directory = directory.as_ref(); 285 285 - let mut index_file = gix_tempfile::new(directory, ContainingDirectory::Exists, AutoRemove::Tempfile)?; 311 311 + let mut index_file = gix_tempfile::new( 312 312 + directory, 313 313 + ContainingDirectory::Exists, 314 314 + AutoRemove::Tempfile, 315 315 + )?; 286 316 287 317 let outcome = crate::index::write_data_iter_to_stream( 288 318 index_kind, ··· 308 338 keep_path: None, 309 339 } 310 340 } else { 311 311 - let data_path = directory.join(format!("pack-{}.pack", outcome.data_hash.to_hex())); 341 341 + let data_path = 342 342 + directory.join(format!("pack-{}.pack", outcome.data_hash.to_hex())); 312 343 let index_path = data_path.with_extension("idx"); 313 344 let keep_path = if data_path.is_file() { 314 345 // avoid trying to overwrite existing files, we know they have the same content

+2 -1

knot2/third_party/gix-pack/src/bundle/write/types.rs

View file Reviewed

··· 62 62 } 63 63 } 64 64 65 65 - pub(crate) type SharedTempFile = Arc<parking_lot::Mutex<std::io::BufWriter<gix_tempfile::Handle<Writable>>>>; 65 65 + pub(crate) type SharedTempFile = 66 66 + Arc<parking_lot::Mutex<std::io::BufWriter<gix_tempfile::Handle<Writable>>>>; 66 67 67 68 pub(crate) struct PassThrough<R> { 68 69 pub reader: R,

+18 -10

knot2/third_party/gix-pack/src/cache/delta/from_offsets.rs

View file Reviewed

··· 14 14 #[allow(missing_docs)] 15 15 pub enum Error { 16 16 #[error("{message}")] 17 17 - Io { source: io::Error, message: &'static str }, 17 17 + Io { 18 18 + source: io::Error, 19 19 + message: &'static str, 20 20 + }, 18 21 #[error(transparent)] 19 22 Header(#[from] crate::data::header::decode::Error), 20 23 #[error("Could find object with id {id} in this pack. Thin packs are not supported")] ··· 86 89 if let Some(previous_offset) = previous_cursor_position { 87 90 Self::advance_cursor_to_pack_offset(&mut r, pack_offset, previous_offset)?; 88 91 } 89 89 - let entry = crate::data::Entry::from_read(&mut r, pack_offset, hash_len).map_err(|err| Error::Io { 90 90 - source: err, 91 91 - message: "EOF while parsing header", 92 92 - })?; 92 92 + let entry = 93 93 + crate::data::Entry::from_read(&mut r, pack_offset, hash_len).map_err(|err| { 94 94 + Error::Io { 95 95 + source: err, 96 96 + message: "EOF while parsing header", 97 97 + } 98 98 + })?; 93 99 previous_cursor_position = Some(pack_offset + entry.header_size() as u64); 94 100 95 101 use crate::data::entry::Header::*; ··· 101 107 resolve_in_pack_id(base_id.as_ref()) 102 108 .ok_or(Error::UnresolvedRefDelta { id: base_id }) 103 109 .and_then(|base_pack_offset| { 104 104 - tree.add_child(base_pack_offset, pack_offset, data).map_err(Into::into) 110 110 + tree.add_child(base_pack_offset, pack_offset, data) 111 111 + .map_err(Into::into) 105 112 })?; 106 113 } 107 114 OfsDelta { base_distance } => { ··· 151 158 // SAFETY: bytes_to_skip <= buf.len() <= usize::MAX 152 159 r.consume(bytes_to_skip as usize); 153 160 } else { 154 154 - r.seek(SeekFrom::Start(pack_offset)).map_err(|err| Error::Io { 155 155 - source: err, 156 156 - message: "seek to next entry", 157 157 - })?; 161 161 + r.seek(SeekFrom::Start(pack_offset)) 162 162 + .map_err(|err| Error::Io { 163 163 + source: err, 164 164 + message: "seek to next entry", 165 165 + })?; 158 166 } 159 167 Ok(()) 160 168 }

+5 -2

knot2/third_party/gix-pack/src/cache/delta/traverse/mod.rs

View file Reviewed

··· 1 1 use std::os::unix::fs::FileExt; 2 2 - use std::sync::{Arc, Mutex}; 3 2 use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; 3 3 + use std::sync::{Arc, Mutex}; 4 4 5 5 use gix_features::{ 6 6 parallel::in_parallel_with_slice, ··· 67 67 fn spill(&self, bytes: &[u8]) -> std::io::Result<SpillRef> { 68 68 let len = bytes.len(); 69 69 let offset = { 70 70 - let mut cursor = self.write_cursor.lock().expect("base spill cursor poisoned"); 70 70 + let mut cursor = self 71 71 + .write_cursor 72 72 + .lock() 73 73 + .expect("base spill cursor poisoned"); 71 74 let offset = *cursor; 72 75 *cursor += len as u64; 73 76 offset

+10 -3

knot2/third_party/gix-pack/src/cache/delta/traverse/resolve.rs

View file Reviewed

··· 24 24 25 25 fn enforce_budget(spill: &super::BaseSpill, stack: &mut [Pending]) -> Result<(), Error> { 26 26 (0..stack.len()).try_for_each(|index| -> Result<(), Error> { 27 27 - if spill.over_budget() && stack[index].spill_ref.is_none() && !stack[index].base_bytes.is_empty() 27 27 + if spill.over_budget() 28 28 + && stack[index].spill_ref.is_none() 29 29 + && !stack[index].base_bytes.is_empty() 28 30 { 29 31 let bytes = std::mem::take(&mut stack[index].base_bytes); 30 30 - let sref = spill.spill(&bytes).map_err(|source| Error::BaseSpill { source })?; 32 32 + let sref = spill 33 33 + .spill(&bytes) 34 34 + .map_err(|source| Error::BaseSpill { source })?; 31 35 stack[index].spill_ref = Some(sref); 32 36 } 33 37 Ok(()) 34 38 }) 35 39 } 36 40 37 37 - fn restore_base_bytes(spill: Option<&super::BaseSpill>, pending: &mut Pending) -> Result<(), Error> { 41 41 + fn restore_base_bytes( 42 42 + spill: Option<&super::BaseSpill>, 43 43 + pending: &mut Pending, 44 44 + ) -> Result<(), Error> { 38 45 if let Some(spill) = spill { 39 46 match pending.spill_ref.take() { 40 47 Some(sref) => spill

+5 -1

knot2/third_party/gix-pack/src/cache/delta/tree.rs

View file Reviewed

··· 8 8 } 9 9 10 10 impl<T> Item<T> { 11 11 - pub(crate) fn new(offset: crate::data::Offset, next_offset: crate::data::Offset, data: T) -> Self { 11 11 + pub(crate) fn new( 12 12 + offset: crate::data::Offset, 13 13 + next_offset: crate::data::Offset, 14 14 + data: T, 15 15 + ) -> Self { 12 16 Item { 13 17 offset, 14 18 next_offset,

+63 -12

knot2/third_party/gix-pack/src/cache/lru.rs

View file Reviewed

··· 37 37 pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { 38 38 MemoryCappedHashmap { 39 39 inner: clru::CLruCache::with_config( 40 40 - clru::CLruCacheConfig::new(NonZeroUsize::new(memory_cap_in_bytes).expect("non zero")) 41 41 - .with_scale(CustomScale), 40 40 + clru::CLruCacheConfig::new( 41 41 + NonZeroUsize::new(memory_cap_in_bytes).expect("non zero"), 42 42 + ) 43 43 + .with_scale(CustomScale), 42 44 ), 43 45 free_list: Vec::new(), 44 44 - debug: gix_features::cache::Debug::new(format!("MemoryCappedHashmap({memory_cap_in_bytes}B)")), 46 46 + debug: gix_features::cache::Debug::new(format!( 47 47 + "MemoryCappedHashmap({memory_cap_in_bytes}B)" 48 48 + )), 45 49 } 46 50 } 47 51 } 48 52 49 53 impl DecodeEntry for MemoryCappedHashmap { 50 50 - fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { 54 54 + fn put( 55 55 + &mut self, 56 56 + pack_id: u32, 57 57 + offset: u64, 58 58 + data: &[u8], 59 59 + kind: gix_object::Kind, 60 60 + compressed_size: usize, 61 61 + ) { 51 62 self.debug.put(); 52 52 - let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) else { 63 63 + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) 64 64 + else { 53 65 return; 54 66 }; 55 67 let res = self.inner.put_with_weight( ··· 67 79 } 68 80 } 69 81 70 70 - fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec<u8>) -> Option<(gix_object::Kind, usize)> { 82 82 + fn get( 83 83 + &mut self, 84 84 + pack_id: u32, 85 85 + offset: u64, 86 86 + out: &mut Vec<u8>, 87 87 + ) -> Option<(gix_object::Kind, usize)> { 71 88 let res = self.inner.get(&(pack_id, offset)).and_then(|e| { 72 89 set_vec_to_slice(out, &e.data)?; 73 90 Some((e.kind, e.compressed_size)) ··· 118 135 last_evicted: Vec::new(), 119 136 debug: gix_features::cache::Debug::new(format!("StaticLinkedList<{SIZE}>")), 120 137 mem_used: 0, 121 121 - mem_limit: if mem_limit == 0 { usize::MAX } else { mem_limit }, 138 138 + mem_limit: if mem_limit == 0 { 139 139 + usize::MAX 140 140 + } else { 141 141 + mem_limit 142 142 + }, 122 143 } 123 144 } 124 145 } ··· 130 151 } 131 152 132 153 impl<const SIZE: usize> DecodeEntry for StaticLinkedList<SIZE> { 133 133 - fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { 154 154 + fn put( 155 155 + &mut self, 156 156 + pack_id: u32, 157 157 + offset: u64, 158 158 + data: &[u8], 159 159 + kind: gix_object::Kind, 160 160 + compressed_size: usize, 161 161 + ) { 134 162 // We cannot possibly hold this much. 135 163 if data.len() > self.mem_limit { 136 164 return; ··· 168 196 } 169 197 } 170 198 171 171 - fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec<u8>) -> Option<(gix_object::Kind, usize)> { 199 199 + fn get( 200 200 + &mut self, 201 201 + pack_id: u32, 202 202 + offset: u64, 203 203 + out: &mut Vec<u8>, 204 204 + ) -> Option<(gix_object::Kind, usize)> { 172 205 let res = self.inner.lookup(|e: &mut Entry| { 173 206 if e.pack_id == pack_id && e.offset == offset { 174 207 set_vec_to_slice(&mut *out, &e.data)?; ··· 217 250 assert_eq!(c.inner.len(), 10); 218 251 assert_eq!(c.last_evicted.len(), 0); 219 252 220 220 - c.put(0, 0, &(0..20).collect::<Vec<_>>(), gix_object::Kind::Blob, 1); 253 253 + c.put( 254 254 + 0, 255 255 + 0, 256 256 + &(0..20).collect::<Vec<_>>(), 257 257 + gix_object::Kind::Blob, 258 258 + 1, 259 259 + ); 221 260 assert_eq!(c.inner.len(), 10); 222 261 assert_eq!(c.mem_used, 80 + 20); 223 262 assert_eq!(c.last_evicted.len(), 1); 224 263 225 225 - c.put(0, 0, &(0..50).collect::<Vec<_>>(), gix_object::Kind::Blob, 1); 264 264 + c.put( 265 265 + 0, 266 266 + 0, 267 267 + &(0..50).collect::<Vec<_>>(), 268 268 + gix_object::Kind::Blob, 269 269 + 1, 270 270 + ); 226 271 assert_eq!(c.inner.len(), 1, "cache clearance wasn't necessary"); 227 272 assert_eq!(c.last_evicted.len(), 0, "the free list was cleared"); 228 273 assert_eq!(c.mem_used, 50); 229 274 230 230 - c.put(0, 0, &(0..101).collect::<Vec<_>>(), gix_object::Kind::Blob, 1); 275 275 + c.put( 276 276 + 0, 277 277 + 0, 278 278 + &(0..101).collect::<Vec<_>>(), 279 279 + gix_object::Kind::Blob, 280 280 + 1, 281 281 + ); 231 282 assert_eq!( 232 283 c.inner.len(), 233 284 1,

+31 -5

knot2/third_party/gix-pack/src/cache/mod.rs

View file Reviewed

··· 9 9 /// Store a fully decoded object at `offset` of `kind` with `compressed_size` and `data` in the cache. 10 10 /// 11 11 /// It is up to the cache implementation whether that actually happens or not. 12 12 - fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize); 12 12 + fn put( 13 13 + &mut self, 14 14 + pack_id: u32, 15 15 + offset: u64, 16 16 + data: &[u8], 17 17 + kind: gix_object::Kind, 18 18 + compressed_size: usize, 19 19 + ); 13 20 /// Attempt to fetch the object at `offset` and store its decoded bytes in `out`, as previously stored with [`DecodeEntry::put()`], and return 14 21 /// its (object `kind`, `decompressed_size`) 15 15 - fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec<u8>) -> Option<(gix_object::Kind, usize)>; 22 22 + fn get( 23 23 + &mut self, 24 24 + pack_id: u32, 25 25 + offset: u64, 26 26 + out: &mut Vec<u8>, 27 27 + ) -> Option<(gix_object::Kind, usize)>; 16 28 } 17 29 18 30 /// A cache that stores nothing and retrieves nothing, thus it _never_ caches. ··· 20 32 pub struct Never; 21 33 22 34 impl DecodeEntry for Never { 23 23 - fn put(&mut self, _pack_id: u32, _offset: u64, _data: &[u8], _kind: gix_object::Kind, _compressed_size: usize) {} 24 24 - fn get(&mut self, _pack_id: u32, _offset: u64, _out: &mut Vec<u8>) -> Option<(gix_object::Kind, usize)> { 35 35 + fn put( 36 36 + &mut self, 37 37 + _pack_id: u32, 38 38 + _offset: u64, 39 39 + _data: &[u8], 40 40 + _kind: gix_object::Kind, 41 41 + _compressed_size: usize, 42 42 + ) { 43 43 + } 44 44 + fn get( 45 45 + &mut self, 46 46 + _pack_id: u32, 47 47 + _offset: u64, 48 48 + _out: &mut Vec<u8>, 49 49 + ) -> Option<(gix_object::Kind, usize)> { 25 50 None 26 51 } 27 52 } 28 53 29 54 impl<T: DecodeEntry + ?Sized> DecodeEntry for Box<T> { 30 55 fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: Kind, compressed_size: usize) { 31 31 - self.deref_mut().put(pack_id, offset, data, kind, compressed_size); 56 56 + self.deref_mut() 57 57 + .put(pack_id, offset, data, kind, compressed_size); 32 58 } 33 59 34 60 fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec<u8>) -> Option<(Kind, usize)> {

+10 -5

knot2/third_party/gix-pack/src/cache/object.rs

View file Reviewed

··· 42 42 pub fn new(memory_cap_in_bytes: usize) -> MemoryCappedHashmap { 43 43 MemoryCappedHashmap { 44 44 inner: clru::CLruCache::with_config( 45 45 - clru::CLruCacheConfig::new(NonZeroUsize::new(memory_cap_in_bytes).expect("non zero")) 46 46 - .with_hasher(gix_hashtable::hash::Builder) 47 47 - .with_scale(CustomScale), 45 45 + clru::CLruCacheConfig::new( 46 46 + NonZeroUsize::new(memory_cap_in_bytes).expect("non zero"), 47 47 + ) 48 48 + .with_hasher(gix_hashtable::hash::Builder) 49 49 + .with_scale(CustomScale), 48 50 ), 49 51 free_list: Vec::new(), 50 50 - debug: gix_features::cache::Debug::new(format!("MemoryCappedObjectHashmap({memory_cap_in_bytes}B)")), 52 52 + debug: gix_features::cache::Debug::new(format!( 53 53 + "MemoryCappedObjectHashmap({memory_cap_in_bytes}B)" 54 54 + )), 51 55 } 52 56 } 53 57 } ··· 56 60 /// Put the object going by `id` of `kind` with `data` into the cache. 57 61 fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]) { 58 62 self.debug.put(); 59 59 - let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) else { 63 63 + let Some(data) = set_vec_to_slice(self.free_list.pop().unwrap_or_default(), data) 64 64 + else { 60 65 return; 61 66 }; 62 67 let res = self.inner.put_with_weight(id, Entry { data, kind });

+5 -3

knot2/third_party/gix-pack/src/data/delta.rs

View file Reviewed

··· 79 79 size = 0x10000; // 65536 80 80 } 81 81 let ofs = ofs as usize; 82 82 - let end = ofs.checked_add(size as usize).ok_or(apply::Error::Corrupt { 83 83 - message: "delta copy range overflows", 84 84 - })?; 82 82 + let end = ofs 83 83 + .checked_add(size as usize) 84 84 + .ok_or(apply::Error::Corrupt { 85 85 + message: "delta copy range overflows", 86 86 + })?; 85 87 std::io::Write::write( 86 88 &mut target, 87 89 base.get(ofs..end).ok_or(apply::Error::Corrupt {

+28 -12

knot2/third_party/gix-pack/src/data/entry/decode.rs

View file Reviewed

··· 24 24 /// # Panics 25 25 /// 26 26 /// If we cannot understand the header, garbage data is likely to trigger this. 27 27 - pub fn from_bytes(d: &[u8], pack_offset: data::Offset, hash_len: usize) -> Result<data::Entry, Error> { 27 27 + pub fn from_bytes( 28 28 + d: &[u8], 29 29 + pack_offset: data::Offset, 30 30 + hash_len: usize, 31 31 + ) -> Result<data::Entry, Error> { 28 32 let (type_id, size, mut consumed) = parse_header_info(d)?; 29 33 30 34 use crate::data::entry::Header::*; ··· 39 43 } 40 44 REF_DELTA => { 41 45 let delta = RefDelta { 42 42 - base_id: gix_hash::ObjectId::from_bytes_or_panic(d.get(consumed..consumed + hash_len).ok_or( 43 43 - Error::Corrupt { 46 46 + base_id: gix_hash::ObjectId::from_bytes_or_panic( 47 47 + d.get(consumed..consumed + hash_len).ok_or(Error::Corrupt { 44 48 message: "ref-delta base object id", 45 45 - }, 46 46 - )?), 49 49 + })?, 50 50 + ), 47 51 }; 48 52 consumed += hash_len; 49 53 delta ··· 62 66 } 63 67 64 68 /// Instantiate an `Entry` from the reader `r`, providing the `pack_offset` to allow tracking the start of the entry data section. 65 65 - pub fn from_read(r: &mut dyn io::Read, pack_offset: data::Offset, hash_len: usize) -> io::Result<data::Entry> { 69 69 + pub fn from_read( 70 70 + r: &mut dyn io::Read, 71 71 + pack_offset: data::Offset, 72 72 + hash_len: usize, 73 73 + ) -> io::Result<data::Entry> { 66 74 let (type_id, size, mut consumed) = streaming_parse_header_info(r)?; 67 75 68 76 use crate::data::entry::Header::*; ··· 90 98 TREE => Tree, 91 99 COMMIT => Commit, 92 100 TAG => Tag, 93 93 - other => return Err(io::Error::other(format!("Object type {other} is unsupported"))), 101 101 + other => { 102 102 + return Err(io::Error::other(format!( 103 103 + "Object type {other} is unsupported" 104 104 + ))); 105 105 + } 94 106 }; 95 107 Ok(data::Entry { 96 108 header: object, ··· 115 127 i += 1; 116 128 let component = u64::from(c & 0b0111_1111) 117 129 .checked_shl(shift) 118 118 - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed"))?; 119 119 - size = size 120 120 - .checked_add(component) 121 121 - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed"))?; 130 130 + .ok_or_else(|| { 131 131 + io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed") 132 132 + })?; 133 133 + size = size.checked_add(component).ok_or_else(|| { 134 134 + io::Error::new(io::ErrorKind::InvalidData, "pack entry header overflowed") 135 135 + })?; 122 136 shift += 7; 123 137 } 124 138 if i != encoded_pack_entry_header_size(size) { ··· 145 159 message: "pack entry header continuation byte", 146 160 })?; 147 161 i += 1; 148 148 - let component = u64::from(c & 0b0111_1111).checked_shl(shift).ok_or(Error::Overflow)?; 162 162 + let component = u64::from(c & 0b0111_1111) 163 163 + .checked_shl(shift) 164 164 + .ok_or(Error::Overflow)?; 149 165 size = size.checked_add(component).ok_or(Error::Overflow)?; 150 166 shift += 7; 151 167 }

+18 -4

knot2/third_party/gix-pack/src/data/entry/header.rs

View file Reviewed

··· 39 39 40 40 impl Header { 41 41 /// Subtract `distance` from `pack_offset` safely without the chance for overflow or no-ops if `distance` is 0. 42 42 - pub fn verified_base_pack_offset(pack_offset: data::Offset, distance: u64) -> Option<data::Offset> { 42 42 + pub fn verified_base_pack_offset( 43 43 + pack_offset: data::Offset, 44 44 + distance: u64, 45 45 + ) -> Option<data::Offset> { 43 46 if distance == 0 { 44 47 return None; 45 48 } ··· 83 86 /// 84 87 /// Returns the amount of bytes written to `out`. 85 88 /// `decompressed_size_in_bytes` is the full size in bytes of the object that this header represents 86 86 - pub fn write_to(&self, decompressed_size_in_bytes: u64, out: &mut dyn io::Write) -> io::Result<usize> { 89 89 + pub fn write_to( 90 90 + &self, 91 91 + decompressed_size_in_bytes: u64, 92 92 + out: &mut dyn io::Write, 93 93 + ) -> io::Result<usize> { 87 94 let mut size = decompressed_size_in_bytes; 88 95 let mut written = 1; 89 96 let mut c: u8 = (self.as_type_id() << 4) | (size as u8 & 0b0000_1111); ··· 133 140 *out = 0b1000_0000 | (n as u8 & 0b0111_1111); 134 141 bytes_written += 1; 135 142 } 136 136 - debug_assert_eq!(n, 0, "BUG: buffer must be large enough to hold a 64 bit integer"); 143 143 + debug_assert_eq!( 144 144 + n, 0, 145 145 + "BUG: buffer must be large enough to hold a 64 bit integer" 146 146 + ); 137 147 &buf[buf.len() - bytes_written..] 138 148 } 139 149 ··· 145 155 fn leb64_encode_max_int() { 146 156 let mut buf = [0u8; 10]; 147 157 let buf = leb64_encode(u64::MAX, &mut buf); 148 148 - assert_eq!(buf.len(), 10, "10 bytes should be used when 64bits are encoded"); 158 158 + assert_eq!( 159 159 + buf.len(), 160 160 + 10, 161 161 + "10 bytes should be used when 64bits are encoded" 162 162 + ); 149 163 } 150 164 }

+46 -21

knot2/third_party/gix-pack/src/data/file/decode/entry.rs

View file Reviewed

··· 61 61 object_size: 0, 62 62 } 63 63 } 64 64 - fn from_object_entry(kind: gix_object::Kind, entry: &data::Entry, compressed_size: usize) -> Self { 64 64 + fn from_object_entry( 65 65 + kind: gix_object::Kind, 66 66 + entry: &data::Entry, 67 67 + compressed_size: usize, 68 68 + ) -> Self { 65 69 Self { 66 70 kind, 67 71 num_deltas: 0, ··· 89 93 inflate: &mut zlib::Inflate, 90 94 out: &mut [u8], 91 95 ) -> Result<usize, Error> { 92 92 - let size: usize = entry.decompressed_size.try_into().map_err(|_| Error::OutOfMemory)?; 96 96 + let size: usize = entry 97 97 + .decompressed_size 98 98 + .try_into() 99 99 + .map_err(|_| Error::OutOfMemory)?; 93 100 if out.len() < size { 94 101 return Err(Error::OutOfMemory); 95 102 } ··· 109 116 110 117 let window = (self.data_len() - pack_offset).min(self.hash_len + 32); 111 118 let mut header = vec![0u8; window]; 112 112 - self.read_exact_at(pack_offset, &mut header) 113 113 - .map_err(|_| data::entry::decode::Error::Corrupt { 119 119 + self.read_exact_at(pack_offset, &mut header).map_err(|_| { 120 120 + data::entry::decode::Error::Corrupt { 114 121 message: "failed to read entry header from pack data", 115 115 - })?; 122 122 + } 123 123 + })?; 116 124 data::Entry::from_bytes(&header, offset, self.hash_len) 117 125 } 118 126 ··· 166 174 inflate: &mut zlib::Inflate, 167 175 out: &mut [u8], 168 176 ) -> Result<(zlib::Status, usize, usize), Error> { 169 169 - let offset: usize = data_offset.try_into().expect("offset representable by machine"); 177 177 + let offset: usize = data_offset 178 178 + .try_into() 179 179 + .expect("offset representable by machine"); 170 180 if offset >= self.data_len() { 171 181 return Err(data::entry::decode::Error::Corrupt { 172 182 message: "an entry data offset pointing beyond pack data", ··· 179 189 let mut in_pos = offset; 180 190 let status = loop { 181 191 let avail = (self.data_len() - in_pos).min(chunk.len()); 182 182 - self.read_exact_at(in_pos, &mut chunk[..avail]).map_err(|_| { 183 183 - Error::from(data::entry::decode::Error::Corrupt { 184 184 - message: "failed to read pack entry data", 185 185 - }) 186 186 - })?; 192 192 + self.read_exact_at(in_pos, &mut chunk[..avail]) 193 193 + .map_err(|_| { 194 194 + Error::from(data::entry::decode::Error::Corrupt { 195 195 + message: "failed to read pack entry data", 196 196 + }) 197 197 + })?; 187 198 let out_pos = inflate.state.total_out() as usize; 188 199 let before_in = inflate.state.total_in(); 189 200 let status = inflate 190 201 .state 191 191 - .decompress(&chunk[..avail], &mut out[out_pos..], zlib::FlushDecompress::None) 202 202 + .decompress( 203 203 + &chunk[..avail], 204 204 + &mut out[out_pos..], 205 205 + zlib::FlushDecompress::None, 206 206 + ) 192 207 .map_err(|err| Error::from(zlib::inflate::Error::from(err)))?; 193 208 let advanced_in = inflate.state.total_in() != before_in; 194 209 let advanced_out = inflate.state.total_out() as usize != out_pos; ··· 245 260 ) 246 261 }) 247 262 } 248 248 - OfsDelta { .. } | RefDelta { .. } => self.resolve_deltas(entry, resolve, inflate, out, delta_cache), 263 263 + OfsDelta { .. } | RefDelta { .. } => { 264 264 + self.resolve_deltas(entry, resolve, inflate, out, delta_cache) 265 265 + } 249 266 } 250 267 } 251 268 ··· 332 349 // First pass will decompress all delta data and keep it in our output buffer 333 350 // [<possibly resolved base object>]<delta-1..delta-n>... 334 351 // so that we can find the biggest result size. 335 335 - let total_delta_data_size: usize = total_delta_data_size.try_into().map_err(|_| Error::OutOfMemory)?; 352 352 + let total_delta_data_size: usize = total_delta_data_size 353 353 + .try_into() 354 354 + .map_err(|_| Error::OutOfMemory)?; 336 355 337 356 let chain_len = chain.len(); 338 357 let (first_buffer_end, second_buffer_end) = { ··· 351 370 let mut relative_delta_start = 0; 352 371 let mut biggest_result_size = 0; 353 372 for (delta_idx, delta) in chain.iter_mut().rev().enumerate() { 354 354 - let (consumed_from_data_offset, consumed_out) = self.decompress_complete_entry_from_data_offset( 355 355 - delta.data_offset, 356 356 - inflate, 357 357 - &mut instructions[..delta.decompressed_size], 358 358 - )?; 373 373 + let (consumed_from_data_offset, consumed_out) = self 374 374 + .decompress_complete_entry_from_data_offset( 375 375 + delta.data_offset, 376 376 + inflate, 377 377 + &mut instructions[..delta.decompressed_size], 378 378 + )?; 359 379 let is_last_delta_to_be_applied = delta_idx + 1 == chain_len; 360 380 if is_last_delta_to_be_applied { 361 381 consumed_input = Some(consumed_from_data_offset); ··· 445 465 if delta_idx + 1 == chain_len { 446 466 last_result_size = Some(result_size); 447 467 } 448 448 - delta::apply(&source_buf[..base_size], &mut target_buf[..result_size], data)?; 468 468 + delta::apply( 469 469 + &source_buf[..base_size], 470 470 + &mut target_buf[..result_size], 471 471 + data, 472 472 + )?; 449 473 // use the target as source for the next delta 450 474 std::mem::swap(&mut source_buf, &mut target_buf); 451 475 } ··· 466 490 debug_assert!(out.len() >= last_result_size); 467 491 out.truncate(last_result_size); 468 492 469 469 - let object_kind = object_kind.expect("a base object as root of any delta chain that we are here to resolve"); 493 493 + let object_kind = object_kind 494 494 + .expect("a base object as root of any delta chain that we are here to resolve"); 470 495 let consumed_input = consumed_input.expect("at least one decompressed delta object"); 471 496 cache.put( 472 497 self.id,

+19 -7

knot2/third_party/gix-pack/src/data/file/decode/header.rs

View file Reviewed

··· 63 63 Tree | Blob | Commit | Tag => { 64 64 return Ok(Outcome { 65 65 kind: entry.header.as_kind().expect("always valid for non-refs"), 66 66 - object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size), 66 66 + object_size: first_delta_decompressed_size 67 67 + .unwrap_or(entry.decompressed_size), 67 68 num_deltas, 68 69 }); 69 70 } 70 71 OfsDelta { base_distance } => { 71 72 num_deltas += 1; 72 73 if first_delta_decompressed_size.is_none() { 73 73 - first_delta_decompressed_size = Some(self.decode_delta_object_size(inflate, &entry)?); 74 74 + first_delta_decompressed_size = 75 75 + Some(self.decode_delta_object_size(inflate, &entry)?); 74 76 } 75 77 entry = self.entry(entry.checked_base_pack_offset(base_distance).ok_or( 76 78 crate::data::entry::decode::Error::Corrupt { ··· 81 83 RefDelta { base_id } => { 82 84 num_deltas += 1; 83 85 if first_delta_decompressed_size.is_none() { 84 84 - first_delta_decompressed_size = Some(self.decode_delta_object_size(inflate, &entry)?); 86 86 + first_delta_decompressed_size = 87 87 + Some(self.decode_delta_object_size(inflate, &entry)?); 85 88 } 86 89 match resolve(base_id.as_ref()) { 87 90 Some(ResolvedBase::InPack(base_entry)) => entry = base_entry, ··· 91 94 }) => { 92 95 return Ok(Outcome { 93 96 kind, 94 94 - object_size: first_delta_decompressed_size.unwrap_or(entry.decompressed_size), 97 97 + object_size: first_delta_decompressed_size 98 98 + .unwrap_or(entry.decompressed_size), 95 99 num_deltas: origin_num_deltas.unwrap_or_default() + num_deltas, 96 100 }); 97 101 } ··· 112 116 /// decompression through `decode_entry()` must still validate that the stream length matches 113 117 /// the pack entry header. 114 118 #[inline] 115 115 - fn decode_delta_object_size(&self, inflate: &mut zlib::Inflate, entry: &data::Entry) -> Result<u64, Error> { 119 119 + fn decode_delta_object_size( 120 120 + &self, 121 121 + inflate: &mut zlib::Inflate, 122 122 + entry: &data::Entry, 123 123 + ) -> Result<u64, Error> { 116 124 let mut buf = [0_u8; 20]; 117 125 let max_size = entry.decompressed_size.min(buf.len() as u64) as usize; 118 118 - let (status, _consumed_in, consumed_out) = 119 119 - self.decompress_entry_from_data_offset_unchecked(entry.data_offset, inflate, &mut buf[..max_size])?; 126 126 + let (status, _consumed_in, consumed_out) = self 127 127 + .decompress_entry_from_data_offset_unchecked( 128 128 + entry.data_offset, 129 129 + inflate, 130 130 + &mut buf[..max_size], 131 131 + )?; 120 132 if status == zlib::Status::StreamEnd { 121 133 if consumed_out as u64 != entry.decompressed_size { 122 134 return Err(data::entry::decode::Error::Corrupt {

+16 -7

knot2/third_party/gix-pack/src/data/file/init.rs

View file Reviewed

··· 11 11 /// 12 12 /// This constructor leaves allocation limiting disabled, allowing allocations of any size dictated by pack data. 13 13 /// Call [`File::with_alloc_limit_bytes()`][crate::data::File::with_alloc_limit_bytes()] before decoding entries from untrusted input. 14 14 - pub fn at(path: impl AsRef<Path>, object_hash: gix_hash::Kind) -> Result<Self, data::header::decode::Error> { 14 14 + pub fn at( 15 15 + path: impl AsRef<Path>, 16 16 + object_hash: gix_hash::Kind, 17 17 + ) -> Result<Self, data::header::decode::Error> { 15 18 Self::at_inner(path.as_ref(), object_hash) 16 19 } 17 20 18 18 - fn at_inner(path: &Path, object_hash: gix_hash::Kind) -> Result<Self, data::header::decode::Error> { 21 21 + fn at_inner( 22 22 + path: &Path, 23 23 + object_hash: gix_hash::Kind, 24 24 + ) -> Result<Self, data::header::decode::Error> { 19 25 use std::os::unix::fs::FileExt; 20 26 21 27 use crate::data::header::N32_SIZE; ··· 32 38 })? 33 39 .len(); 34 40 let pack_len = usize::try_from(pack_len).map_err(|_| { 35 35 - data::header::decode::Error::Corrupt(format!("Pack data of size {pack_len} is too large for this machine")) 41 41 + data::header::decode::Error::Corrupt(format!( 42 42 + "Pack data of size {pack_len} is too large for this machine" 43 43 + )) 36 44 })?; 37 45 if pack_len < N32_SIZE * 3 + hash_len { 38 46 return Err(data::header::decode::Error::Corrupt(format!( ··· 40 48 ))); 41 49 } 42 50 let mut header = [0u8; 12]; 43 43 - file.read_exact_at(&mut header, 0).map_err(|e| data::header::decode::Error::Io { 44 44 - source: e, 45 45 - path: path.to_owned(), 46 46 - })?; 51 51 + file.read_exact_at(&mut header, 0) 52 52 + .map_err(|e| data::header::decode::Error::Io { 53 53 + source: e, 54 54 + path: path.to_owned(), 55 55 + })?; 47 56 let (version, num_objects) = data::header::decode(&header)?; 48 57 let id = gix_features::hash::crc32(path.as_os_str().to_string_lossy().as_bytes()); 49 58 Ok(Self {

+3 -1

knot2/third_party/gix-pack/src/data/file/verify.rs

View file Reviewed

··· 15 15 /// The checksum in the trailer of this pack data file 16 16 pub fn checksum(&self) -> gix_hash::ObjectId { 17 17 let trailer = self 18 18 - .read_span((self.data_len() - self.object_hash.len_in_bytes()) as u64..self.data_len() as u64) 18 18 + .read_span( 19 19 + (self.data_len() - self.object_hash.len_in_bytes()) as u64..self.data_len() as u64, 20 20 + ) 19 21 .expect("pack trailer is within the pack data"); 20 22 gix_hash::ObjectId::from_bytes_or_panic(&trailer) 21 23 }

+3 -1

knot2/third_party/gix-pack/src/data/header.rs

View file Reviewed

··· 6 6 pub fn decode(data: &[u8; 12]) -> Result<(data::Version, u32), decode::Error> { 7 7 let mut ofs = 0; 8 8 if &data[ofs..ofs + b"PACK".len()] != b"PACK" { 9 9 - return Err(decode::Error::Corrupt("Pack data type not recognized".into())); 9 9 + return Err(decode::Error::Corrupt( 10 10 + "Pack data type not recognized".into(), 11 11 + )); 10 12 } 11 13 ofs += N32_SIZE; 12 14 let kind = match crate::read_u32(&data[ofs..ofs + N32_SIZE]) {

+12 -5

knot2/third_party/gix-pack/src/data/input/bytes_to_entries.rs

View file Reviewed

··· 52 52 object_hash: gix_hash::Kind, 53 53 ) -> Result<BytesToEntriesIter<BR>, input::Error> { 54 54 let mut header_data = [0u8; 12]; 55 55 - read.read_exact(&mut header_data).map_err(gix_hash::io::Error::from)?; 55 55 + read.read_exact(&mut header_data) 56 56 + .map_err(gix_hash::io::Error::from)?; 56 57 57 58 let (version, num_objects) = crate::data::header::decode(&header_data)?; 58 59 match version { ··· 101 102 .map_err(gix_hash::io::Error::from)?; 102 103 103 104 // Decompress object to learn its compressed bytes 104 104 - let compressed_buf = self.compressed_buf.take().unwrap_or_else(|| Vec::with_capacity(4096)); 105 105 + let compressed_buf = self 106 106 + .compressed_buf 107 107 + .take() 108 108 + .unwrap_or_else(|| Vec::with_capacity(4096)); 105 109 self.decompressor.reset(); 106 110 let mut decompressed_reader = DecompressRead { 107 111 inner: read_and_pass_to( ··· 115 119 decompressor: &mut self.decompressor, 116 120 }; 117 121 118 118 - let bytes_copied = io::copy(&mut decompressed_reader, &mut io::sink()).map_err(gix_hash::io::Error::from)?; 122 122 + let bytes_copied = io::copy(&mut decompressed_reader, &mut io::sink()) 123 123 + .map_err(gix_hash::io::Error::from)?; 119 124 if bytes_copied != entry.decompressed_size { 120 125 return Err(input::Error::IncompletePack { 121 126 actual: bytes_copied, ··· 273 278 impl crate::data::File { 274 279 /// Returns an iterator over [`Entries`][crate::data::input::Entry], without making use of the memory mapping. 275 280 pub fn streaming_iter(&self) -> Result<BytesToEntriesIter<impl io::BufRead>, input::Error> { 276 276 - let reader = 277 277 - io::BufReader::with_capacity(4096 * 8, fs::File::open(&self.path).map_err(gix_hash::io::Error::from)?); 281 281 + let reader = io::BufReader::with_capacity( 282 282 + 4096 * 8, 283 283 + fs::File::open(&self.path).map_err(gix_hash::io::Error::from)?, 284 284 + ); 278 285 BytesToEntriesIter::new_from_header( 279 286 reader, 280 287 input::Mode::Verify,

+15 -4

knot2/third_party/gix-pack/src/data/input/entries_to_bytes.rs

View file Reviewed

··· 38 38 /// # Panics 39 39 /// 40 40 /// Only [Version::V2](crate::data::Version::V2) is allowed for `version. 41 41 - pub fn new(input: I, output: W, version: crate::data::Version, object_hash: gix_hash::Kind) -> Self { 41 41 + pub fn new( 42 42 + input: I, 43 43 + output: W, 44 44 + version: crate::data::Version, 45 45 + object_hash: gix_hash::Kind, 46 46 + ) -> Self { 42 47 assert!( 43 48 matches!(version, crate::data::Version::V2), 44 49 "currently only pack version 2 can be written", ··· 66 71 self.output.write_all(&header_bytes[..])?; 67 72 } 68 73 self.num_entries += 1; 69 69 - entry.header.write_to(entry.decompressed_size, &mut self.output)?; 74 74 + entry 75 75 + .header 76 76 + .write_to(entry.decompressed_size, &mut self.output)?; 70 77 self.output.write_all( 71 78 entry 72 79 .compressed ··· 76 83 Ok(entry) 77 84 } 78 85 79 79 - fn write_header_and_digest(&mut self, last_entry: Option<&mut input::Entry>) -> Result<(), gix_hash::io::Error> { 86 86 + fn write_header_and_digest( 87 87 + &mut self, 88 88 + last_entry: Option<&mut input::Entry>, 89 89 + ) -> Result<(), gix_hash::io::Error> { 80 90 let header_bytes = crate::data::header::encode(self.data_version, self.num_entries); 81 91 let num_bytes_written = if last_entry.is_some() { 82 92 self.output.stream_position()? ··· 127 137 .next_inner(entry) 128 138 .and_then(|mut entry| { 129 139 if self.input.peek().is_none() { 130 130 - self.write_header_and_digest(Some(&mut entry)).map(|_| entry) 140 140 + self.write_header_and_digest(Some(&mut entry)) 141 141 + .map(|_| entry) 131 142 } else { 132 143 Ok(entry) 133 144 }

+4 -1

knot2/third_party/gix-pack/src/data/input/entry.rs

View file Reviewed

··· 6 6 /// Create a new input entry from a given data `obj` set to be placed at the given `pack_offset`. 7 7 /// 8 8 /// This method is useful when arbitrary base entries are created 9 9 - pub fn from_data_obj(obj: &gix_object::Data<'_>, pack_offset: u64) -> Result<Self, input::Error> { 9 9 + pub fn from_data_obj( 10 10 + obj: &gix_object::Data<'_>, 11 11 + pack_offset: u64, 12 12 + ) -> Result<Self, input::Error> { 10 13 let header = to_header(obj.kind); 11 14 let compressed = compress_data(obj)?; 12 15 let compressed_size = compressed.len() as u64;

+65 -32

knot2/third_party/gix-pack/src/data/input/lookup_ref_delta_objects.rs

View file Reviewed

··· 40 40 41 41 fn shifted_pack_offset(&self, pack_offset: u64) -> u64 { 42 42 let new_ofs = pack_offset as i64 + self.inserted_entries_length_in_bytes; 43 43 - new_ofs.try_into().expect("offset value is never becomes negative") 43 43 + new_ofs 44 44 + .try_into() 45 45 + .expect("offset value is never becomes negative") 44 46 } 45 47 46 48 /// positive `size_change` values mean an object grew or was more commonly, was inserted. Negative values 47 49 /// mean the object shrunk, usually because there header changed from ref-deltas to ofs deltas. 48 48 - fn track_change(&mut self, shifted_pack_offset: u64, pack_offset: u64, size_change: i64, oid: Option<ObjectId>) { 50 50 + fn track_change( 51 51 + &mut self, 52 52 + shifted_pack_offset: u64, 53 53 + pack_offset: u64, 54 54 + size_change: i64, 55 55 + oid: Option<ObjectId>, 56 56 + ) { 49 57 if size_change == 0 { 50 58 return; 51 59 } ··· 60 68 self.inserted_entries_length_in_bytes += size_change; 61 69 } 62 70 63 63 - fn shift_entry_and_point_to_base_by_offset(&mut self, entry: &mut input::Entry, base_distance: u64) { 71 71 + fn shift_entry_and_point_to_base_by_offset( 72 72 + &mut self, 73 73 + entry: &mut input::Entry, 74 74 + base_distance: u64, 75 75 + ) { 64 76 let pack_offset = entry.pack_offset; 65 77 entry.pack_offset = self.shifted_pack_offset(pack_offset); 66 78 entry.header = Header::OfsDelta { base_distance }; ··· 90 102 match self.inner.next() { 91 103 Some(Ok(mut entry)) => match entry.header { 92 104 Header::RefDelta { base_id } => { 93 93 - match self.inserted_entry_length_at_offset.iter().rfind(|e| e.oid == base_id) { 105 105 + match self 106 106 + .inserted_entry_length_at_offset 107 107 + .iter() 108 108 + .rfind(|e| e.oid == base_id) 109 109 + { 94 110 None => { 95 95 - let base_entry = match self.lookup.try_find(&base_id, &mut self.buf).ok()? { 96 96 - Some(obj) => { 97 97 - let current_pack_offset = entry.pack_offset; 98 98 - let mut entry = match input::Entry::from_data_obj(&obj, 0) { 99 99 - Ok(e) => e, 100 100 - Err(err) => return Some(Err(err)), 101 101 - }; 102 102 - entry.pack_offset = self.shifted_pack_offset(current_pack_offset); 103 103 - self.track_change( 104 104 - entry.pack_offset, 105 105 - current_pack_offset, 106 106 - entry.bytes_in_pack() as i64, 107 107 - Some(base_id), 108 108 - ); 109 109 - entry 110 110 - } 111 111 - None => { 112 112 - self.error = true; 113 113 - return Some(Err(input::Error::NotFound { object_id: base_id })); 114 114 - } 115 115 - }; 111 111 + let base_entry = 112 112 + match self.lookup.try_find(&base_id, &mut self.buf).ok()? { 113 113 + Some(obj) => { 114 114 + let current_pack_offset = entry.pack_offset; 115 115 + let mut entry = match input::Entry::from_data_obj(&obj, 0) { 116 116 + Ok(e) => e, 117 117 + Err(err) => return Some(Err(err)), 118 118 + }; 119 119 + entry.pack_offset = 120 120 + self.shifted_pack_offset(current_pack_offset); 121 121 + self.track_change( 122 122 + entry.pack_offset, 123 123 + current_pack_offset, 124 124 + entry.bytes_in_pack() as i64, 125 125 + Some(base_id), 126 126 + ); 127 127 + entry 128 128 + } 129 129 + None => { 130 130 + self.error = true; 131 131 + return Some(Err(input::Error::NotFound { 132 132 + object_id: base_id, 133 133 + })); 134 134 + } 135 135 + }; 116 136 117 137 { 118 118 - self.shift_entry_and_point_to_base_by_offset(&mut entry, base_entry.bytes_in_pack()); 138 138 + self.shift_entry_and_point_to_base_by_offset( 139 139 + &mut entry, 140 140 + base_entry.bytes_in_pack(), 141 141 + ); 119 142 self.next_delta = Some(entry); 120 143 } 121 144 Some(Ok(base_entry)) 122 145 } 123 146 Some(base_entry) => { 124 124 - let base_distance = 125 125 - self.shifted_pack_offset(entry.pack_offset) - base_entry.shifted_pack_offset; 147 147 + let base_distance = self.shifted_pack_offset(entry.pack_offset) 148 148 + - base_entry.shifted_pack_offset; 126 149 self.shift_entry_and_point_to_base_by_offset(&mut entry, base_distance); 127 150 Some(Ok(entry)) 128 151 } ··· 154 177 }; 155 178 let new_distance = self 156 179 .shifted_pack_offset(entry.pack_offset) 157 157 - .checked_sub(self.inserted_entry_length_at_offset[index].shifted_pack_offset) 180 180 + .checked_sub( 181 181 + self.inserted_entry_length_at_offset[index] 182 182 + .shifted_pack_offset, 183 183 + ) 158 184 .expect("a base that is behind us in the pack"); 159 159 - self.shift_entry_and_point_to_base_by_offset(&mut entry, new_distance); 185 185 + self.shift_entry_and_point_to_base_by_offset( 186 186 + &mut entry, 187 187 + new_distance, 188 188 + ); 160 189 } 161 190 Err(index) => { 162 162 - let change_since_offset = self.inserted_entry_length_at_offset[index..] 191 191 + let change_since_offset = self.inserted_entry_length_at_offset 192 192 + [index..] 163 193 .iter() 164 194 .map(|c| c.size_change_in_bytes) 165 195 .sum::<i64>(); ··· 168 198 .try_into() 169 199 .expect("it still points behind us") 170 200 }; 171 171 - self.shift_entry_and_point_to_base_by_offset(&mut entry, new_distance); 201 201 + self.shift_entry_and_point_to_base_by_offset( 202 202 + &mut entry, 203 203 + new_distance, 204 204 + ); 172 205 } 173 206 } 174 207 } else {

+3 -1

knot2/third_party/gix-pack/src/data/input/types.rs

View file Reviewed

··· 9 9 PackParse(#[from] crate::data::header::decode::Error), 10 10 #[error("Failed to verify pack checksum in trailer")] 11 11 Verify(#[from] gix_hash::verify::Error), 12 12 - #[error("pack is incomplete: it was decompressed into {actual} bytes but {expected} bytes where expected.")] 12 12 + #[error( 13 13 + "pack is incomplete: it was decompressed into {actual} bytes but {expected} bytes where expected." 14 14 + )] 13 15 IncompletePack { actual: u64, expected: u64 }, 14 16 #[error("The object {object_id} could not be decoded or wasn't found")] 15 17 NotFound { object_id: gix_hash::ObjectId },

+2 -1

knot2/third_party/gix-pack/src/data/mod.rs

View file Reviewed

··· 130 130 131 131 #[allow(missing_docs)] 132 132 pub fn read_into(&self, slice: EntryRange, buf: &mut Vec<u8>) -> bool { 133 133 - let (Ok(start), Ok(end)) = (usize::try_from(slice.start), usize::try_from(slice.end)) else { 133 133 + let (Ok(start), Ok(end)) = (usize::try_from(slice.start), usize::try_from(slice.end)) 134 134 + else { 134 135 return false; 135 136 }; 136 137 if start > end || end > self.len {

+6 -2

knot2/third_party/gix-pack/src/data/output/bytes.rs

View file Reviewed

··· 117 117 }); 118 118 self.written += header 119 119 .write_to(entry.decompressed_size as u64, &mut self.output) 120 120 - .map_err(gix_hash::io::Error::from)? as u64; 120 120 + .map_err(gix_hash::io::Error::from)? 121 121 + as u64; 121 122 self.written += std::io::copy(&mut &*entry.compressed_data, &mut self.output) 122 123 .map_err(gix_hash::io::Error::from)?; 123 124 } ··· 134 135 .write_all(digest.as_slice()) 135 136 .map_err(gix_hash::io::Error::from)?; 136 137 self.written += digest.as_slice().len() as u64; 137 137 - self.output.inner.flush().map_err(gix_hash::io::Error::from)?; 138 138 + self.output 139 139 + .inner 140 140 + .flush() 141 141 + .map_err(gix_hash::io::Error::from)?; 138 142 self.is_done = true; 139 143 self.trailer = Some(digest); 140 144 }

+4 -1

knot2/third_party/gix-pack/src/data/output/count/mod.rs

View file Reviewed

··· 31 31 32 32 impl Count { 33 33 /// Create a new instance from the given `oid` and its corresponding location. 34 34 - pub fn from_data(oid: impl Into<ObjectId>, location: Option<crate::data::entry::Location>) -> Self { 34 34 + pub fn from_data( 35 35 + oid: impl Into<ObjectId>, 36 36 + location: Option<crate::data::entry::Location>, 37 37 + ) -> Self { 35 38 Count { 36 39 id: oid.into(), 37 40 entry_pack_location: PackLocation::LookedUp(location),

+78 -20

knot2/third_party/gix-pack/src/data/output/count/objects/mod.rs

View file Reviewed

··· 31 31 /// * more configuration 32 32 pub fn objects<Find>( 33 33 db: Find, 34 34 - objects_ids: Box<dyn Iterator<Item = Result<ObjectId, Box<dyn std::error::Error + Send + Sync + 'static>>> + Send>, 34 34 + objects_ids: Box< 35 35 + dyn Iterator<Item = Result<ObjectId, Box<dyn std::error::Error + Send + Sync + 'static>>> 36 36 + + Send, 37 37 + >, 35 38 objects: &dyn gix_features::progress::Count, 36 39 should_interrupt: &AtomicBool, 37 40 Options { ··· 46 49 let lower_bound = objects_ids.size_hint().0; 47 50 let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit( 48 51 chunk_size, 49 49 - if lower_bound == 0 { None } else { Some(lower_bound) }, 52 52 + if lower_bound == 0 { 53 53 + None 54 54 + } else { 55 55 + Some(lower_bound) 56 56 + }, 50 57 thread_limit, 51 58 None, 52 59 ); ··· 92 99 /// Like [`objects()`] but using a single thread only to mostly save on the otherwise required overhead. 93 100 pub fn objects_unthreaded( 94 101 db: &dyn crate::Find, 95 95 - object_ids: &mut dyn Iterator<Item = Result<ObjectId, Box<dyn std::error::Error + Send + Sync + 'static>>>, 102 102 + object_ids: &mut dyn Iterator< 103 103 + Item = Result<ObjectId, Box<dyn std::error::Error + Send + Sync + 'static>>, 104 104 + >, 96 105 objects: &dyn gix_features::progress::Count, 97 106 should_interrupt: &AtomicBool, 98 107 input_object_expansion: ObjectExpansion, ··· 137 146 db: &dyn crate::Find, 138 147 input_object_expansion: ObjectExpansion, 139 148 seen_objs: &impl util::InsertImmutable, 140 140 - oids: &mut dyn Iterator<Item = Result<ObjectId, Box<dyn std::error::Error + Send + Sync + 'static>>>, 149 149 + oids: &mut dyn Iterator< 150 150 + Item = Result<ObjectId, Box<dyn std::error::Error + Send + Sync + 'static>>, 151 151 + >, 141 152 buf1: &mut Vec<u8>, 142 153 #[allow(clippy::ptr_arg)] buf2: &mut Vec<u8>, 143 154 objects: &gix_features::progress::AtomicStep, ··· 171 182 let mut id = id.to_owned(); 172 183 173 184 loop { 174 174 - push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false); 185 185 + push_obj_count_unique( 186 186 + &mut out, seen_objs, &id, location, objects, stats, false, 187 187 + ); 175 188 match obj.kind { 176 189 Tree | Blob => break, 177 190 Tag => { ··· 188 201 } 189 202 Commit => { 190 203 let current_tree_iter = { 191 191 - let mut commit_iter = CommitRefIter::from_bytes(obj.data, obj.object_hash); 192 192 - let tree_id = commit_iter.tree_id().expect("every commit has a tree"); 204 204 + let mut commit_iter = 205 205 + CommitRefIter::from_bytes(obj.data, obj.object_hash); 206 206 + let tree_id = 207 207 + commit_iter.tree_id().expect("every commit has a tree"); 193 208 parent_commit_ids.clear(); 194 209 for token in commit_iter { 195 210 match token { 196 196 - Ok(gix_object::commit::ref_iter::Token::Parent { id }) => { 211 211 + Ok(gix_object::commit::ref_iter::Token::Parent { 212 212 + id, 213 213 + }) => { 197 214 parent_commit_ids.push(id); 198 215 } 199 216 Ok(_) => break, ··· 202 219 } 203 220 let (obj, location) = db.find(&tree_id, buf1)?; 204 221 push_obj_count_unique( 205 205 - &mut out, seen_objs, &tree_id, location, objects, stats, true, 222 222 + &mut out, seen_objs, &tree_id, location, objects, stats, 223 223 + true, 206 224 ); 207 225 gix_object::TreeRefIter::from_bytes(obj.data, obj.object_hash) 208 226 }; ··· 222 240 } else { 223 241 for commit_id in &parent_commit_ids { 224 242 let parent_tree_id = { 225 225 - let (parent_commit_obj, location) = db.find(commit_id, buf2)?; 243 243 + let (parent_commit_obj, location) = 244 244 + db.find(commit_id, buf2)?; 226 245 227 246 push_obj_count_unique( 228 228 - &mut out, seen_objs, commit_id, location, objects, stats, true, 247 247 + &mut out, seen_objs, commit_id, location, objects, 248 248 + stats, true, 229 249 ); 230 250 CommitRefIter::from_bytes( 231 251 parent_commit_obj.data, ··· 235 255 .expect("every commit has a tree") 236 256 }; 237 257 let parent_tree = { 238 238 - let (parent_tree_obj, location) = db.find(&parent_tree_id, buf2)?; 258 258 + let (parent_tree_obj, location) = 259 259 + db.find(&parent_tree_id, buf2)?; 239 260 push_obj_count_unique( 240 261 &mut out, 241 262 seen_objs, ··· 266 287 &changes_delegate.objects 267 288 }; 268 289 for id in objects_ref.iter() { 269 269 - out.push(id_to_count(db, buf2, id, objects, stats, allow_pack_lookups)); 290 290 + out.push(id_to_count( 291 291 + db, 292 292 + buf2, 293 293 + id, 294 294 + objects, 295 295 + stats, 296 296 + allow_pack_lookups, 297 297 + )); 270 298 } 271 299 break; 272 300 } ··· 278 306 let mut id = id; 279 307 let mut obj = (obj, location); 280 308 loop { 281 281 - push_obj_count_unique(&mut out, seen_objs, &id, obj.1.clone(), objects, stats, false); 309 309 + push_obj_count_unique( 310 310 + &mut out, 311 311 + seen_objs, 312 312 + &id, 313 313 + obj.1.clone(), 314 314 + objects, 315 315 + stats, 316 316 + false, 317 317 + ); 282 318 match obj.0.kind { 283 319 Tree => { 284 320 traverse_delegate.clear(); 285 321 { 286 322 let objects = ExpandedCountingObjects::new(db, out, objects); 287 323 gix_traverse::tree::breadthfirst( 288 288 - gix_object::TreeRefIter::from_bytes(obj.0.data, obj.0.object_hash), 324 324 + gix_object::TreeRefIter::from_bytes( 325 325 + obj.0.data, 326 326 + obj.0.object_hash, 327 327 + ), 289 328 &mut tree_traversal_state, 290 329 &objects, 291 330 &mut traverse_delegate, ··· 294 333 out = objects.dissolve(stats); 295 334 } 296 335 for id in &traverse_delegate.non_trees { 297 297 - out.push(id_to_count(db, buf1, id, objects, stats, allow_pack_lookups)); 336 336 + out.push(id_to_count( 337 337 + db, 338 338 + buf1, 339 339 + id, 340 340 + objects, 341 341 + stats, 342 342 + allow_pack_lookups, 343 343 + )); 298 344 } 299 345 break; 300 346 } ··· 318 364 } 319 365 } 320 366 } 321 321 - AsIs => push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false), 367 367 + AsIs => { 368 368 + push_obj_count_unique(&mut out, seen_objs, &id, location, objects, stats, false) 369 369 + } 322 370 } 323 371 } 324 372 outcome.total_objects = out.len(); ··· 386 434 } 387 435 388 436 impl gix_object::Find for CountingObjects<'_> { 389 389 - fn try_find<'a>(&self, id: &oid, buffer: &'a mut Vec<u8>) -> Result<Option<Data<'a>>, gix_object::find::Error> { 437 437 + fn try_find<'a>( 438 438 + &self, 439 439 + id: &oid, 440 440 + buffer: &'a mut Vec<u8>, 441 441 + ) -> Result<Option<Data<'a>>, gix_object::find::Error> { 390 442 let res = Ok(self.objects.try_find(id, buffer)?.map(|t| t.0)); 391 443 *self.decoded_objects.borrow_mut() += 1; 392 444 res ··· 424 476 } 425 477 426 478 impl gix_object::Find for ExpandedCountingObjects<'_> { 427 427 - fn try_find<'a>(&self, id: &oid, buffer: &'a mut Vec<u8>) -> Result<Option<Data<'a>>, gix_object::find::Error> { 479 479 + fn try_find<'a>( 480 480 + &self, 481 481 + id: &oid, 482 482 + buffer: &'a mut Vec<u8>, 483 483 + ) -> Result<Option<Data<'a>>, gix_object::find::Error> { 428 484 let maybe_obj = self.objects.try_find(id, buffer)?; 429 485 *self.decoded_objects.borrow_mut() += 1; 430 486 match maybe_obj { ··· 432 488 Some((obj, location)) => { 433 489 self.objects_count.fetch_add(1, Ordering::Relaxed); 434 490 *self.expanded_objects.borrow_mut() += 1; 435 435 - self.out.borrow_mut().push(output::Count::from_data(id, location)); 491 491 + self.out 492 492 + .borrow_mut() 493 493 + .push(output::Count::from_data(id, location)); 436 494 Ok(Some(obj)) 437 495 } 438 496 }

+3 -1

knot2/third_party/gix-pack/src/data/output/count/objects/tree.rs

View file Reviewed

··· 47 47 entry_mode, 48 48 relation: _, 49 49 } 50 50 - | Change::Modification { oid, entry_mode, .. } => { 50 50 + | Change::Modification { 51 51 + oid, entry_mode, .. 52 52 + } => { 51 53 if entry_mode.is_commit() { 52 54 return std::ops::ControlFlow::Continue(()); 53 55 }

+48 -22

knot2/third_party/gix-pack/src/data/output/entry/iter_from_counts.rs

View file Reviewed

··· 62 62 matches!(version, crate::data::Version::V2), 63 63 "currently we can only write version 2" 64 64 ); 65 65 - let (chunk_size, thread_limit, _) = 66 66 - parallel::optimize_chunk_size_and_thread_limit(chunk_size, Some(counts.len()), thread_limit, None); 65 65 + let (chunk_size, thread_limit, _) = parallel::optimize_chunk_size_and_thread_limit( 66 66 + chunk_size, 67 67 + Some(counts.len()), 68 68 + thread_limit, 69 69 + None, 70 70 + ); 67 71 { 68 72 let progress = Arc::new(parking_lot::Mutex::new( 69 73 progress.add_child_with_id("resolving".into(), ProgressId::ResolveCounts.into()), 70 74 )); 71 71 - progress.lock().init(None, gix_features::progress::count("counts")); 75 75 + progress 76 76 + .lock() 77 77 + .init(None, gix_features::progress::count("counts")); 72 78 let enough_counts_present = counts.len() > 4_000; 73 79 let start = std::time::Instant::now(); 74 80 parallel::in_parallel_if( ··· 85 91 use crate::data::output::count::PackLocation::*; 86 92 match count.entry_pack_location { 87 93 LookedUp(_) => continue, 88 88 - NotLookedUp => count.entry_pack_location = LookedUp(db.location_by_oid(&count.id, buf)), 94 94 + NotLookedUp => { 95 95 + count.entry_pack_location = 96 96 + LookedUp(db.location_by_oid(&count.id, buf)) 97 97 + } 89 98 } 90 99 } 91 100 progress.lock().inc_by(chunk_size); ··· 99 108 } 100 109 let counts_range_by_pack_id = match mode { 101 110 Mode::PackCopyAndBaseObjects => { 102 102 - let mut progress = progress.add_child_with_id("sorting".into(), ProgressId::SortEntries.into()); 111 111 + let mut progress = 112 112 + progress.add_child_with_id("sorting".into(), ProgressId::SortEntries.into()); 103 113 progress.init(Some(counts.len()), gix_features::progress::count("counts")); 104 114 let start = std::time::Instant::now(); 105 115 106 116 use crate::data::output::count::PackLocation::*; 107 107 - counts.sort_by(|lhs, rhs| match (&lhs.entry_pack_location, &rhs.entry_pack_location) { 108 108 - (LookedUp(None), LookedUp(None)) => Ordering::Equal, 109 109 - (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater, 110 110 - (LookedUp(None), LookedUp(Some(_))) => Ordering::Less, 111 111 - (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs 112 112 - .pack_id 113 113 - .cmp(&rhs.pack_id) 114 114 - .then(lhs.pack_offset.cmp(&rhs.pack_offset)), 115 115 - (_, _) => unreachable!("counts were resolved beforehand"), 117 117 + counts.sort_by(|lhs, rhs| { 118 118 + match (&lhs.entry_pack_location, &rhs.entry_pack_location) { 119 119 + (LookedUp(None), LookedUp(None)) => Ordering::Equal, 120 120 + (LookedUp(Some(_)), LookedUp(None)) => Ordering::Greater, 121 121 + (LookedUp(None), LookedUp(Some(_))) => Ordering::Less, 122 122 + (LookedUp(Some(lhs)), LookedUp(Some(rhs))) => lhs 123 123 + .pack_id 124 124 + .cmp(&rhs.pack_id) 125 125 + .then(lhs.pack_offset.cmp(&rhs.pack_offset)), 126 126 + (_, _) => unreachable!("counts were resolved beforehand"), 127 127 + } 116 128 }); 117 129 118 130 let mut index: Vec<(u32, std::ops::Range<usize>)> = Vec::new(); 119 119 - let mut chunks_pack_start = counts.partition_point(|e| e.entry_pack_location.is_none()); 131 131 + let mut chunks_pack_start = 132 132 + counts.partition_point(|e| e.entry_pack_location.is_none()); 120 133 let mut slice = &counts[chunks_pack_start..]; 121 134 while !slice.is_empty() { 122 122 - let current_pack_id = slice[0].entry_pack_location.as_ref().expect("packed object").pack_id; 135 135 + let current_pack_id = slice[0] 136 136 + .entry_pack_location 137 137 + .as_ref() 138 138 + .expect("packed object") 139 139 + .pack_id; 123 140 let pack_end = slice.partition_point(|e| { 124 124 - e.entry_pack_location.as_ref().expect("packed object").pack_id == current_pack_id 141 141 + e.entry_pack_location 142 142 + .as_ref() 143 143 + .expect("packed object") 144 144 + .pack_id 145 145 + == current_pack_id 125 146 }); 126 126 - index.push((current_pack_id, chunks_pack_start..chunks_pack_start + pack_end)); 147 147 + index.push(( 148 148 + current_pack_id, 149 149 + chunks_pack_start..chunks_pack_start + pack_end, 150 150 + )); 127 151 slice = &slice[pack_end..]; 128 152 chunks_pack_start += pack_end; 129 153 } ··· 147 171 move |n| { 148 172 ( 149 173 Vec::new(), // object data buffer 150 150 - progress 151 151 - .lock() 152 152 - .add_child_with_id(format!("thread {n}"), gix_features::progress::UNKNOWN), 174 174 + progress.lock().add_child_with_id( 175 175 + format!("thread {n}"), 176 176 + gix_features::progress::UNKNOWN, 177 177 + ), 153 178 ) 154 179 } 155 180 }, 156 181 { 157 182 let counts = Arc::clone(&counts); 158 158 - move |(chunk_id, chunk_range): (SequenceId, std::ops::Range<usize>), (buf, progress)| { 183 183 + move |(chunk_id, chunk_range): (SequenceId, std::ops::Range<usize>), 184 184 + (buf, progress)| { 159 185 let mut out = Vec::new(); 160 186 let chunk = &counts[chunk_range]; 161 187 let mut stats = Outcome::default();

+14 -5

knot2/third_party/gix-pack/src/data/output/entry/mod.rs

View file Reviewed

··· 75 75 } 76 76 77 77 let pack_offset_must_be_zero = 0; 78 78 - let pack_entry = match data::Entry::from_bytes(&entry.data, pack_offset_must_be_zero, count.id.as_slice().len()) 79 79 - { 78 78 + let pack_entry = match data::Entry::from_bytes( 79 79 + &entry.data, 80 80 + pack_offset_must_be_zero, 81 81 + count.id.as_slice().len(), 82 82 + ) { 80 83 Ok(e) => e, 81 84 Err(err) => return Some(Err(err.into())), 82 85 }; ··· 122 125 entry.data.copy_within(pack_entry.data_offset as usize.., 0); 123 126 entry.data.resize( 124 127 entry.data.len() 125 125 - - usize::try_from(pack_entry.data_offset).expect("offset representable as usize"), 128 128 + - usize::try_from(pack_entry.data_offset) 129 129 + .expect("offset representable as usize"), 126 130 0, 127 131 ); 128 132 entry.data ··· 142 146 if let Err(err) = std::io::copy(&mut &*obj.data, &mut out) { 143 147 match err.kind() { 144 148 std::io::ErrorKind::Other => return Err(Error::ZlibDeflate(err)), 145 145 - err => unreachable!("Should never see other errors than zlib, but got {:?}", err), 149 149 + err => unreachable!( 150 150 + "Should never see other errors than zlib, but got {:?}", 151 151 + err 152 152 + ), 146 153 } 147 154 } 148 155 out.flush()?; ··· 177 184 Tag => data::entry::Header::Tag, 178 185 } 179 186 } 180 180 - DeltaOid { id } => data::entry::Header::RefDelta { base_id: id.to_owned() }, 187 187 + DeltaOid { id } => data::entry::Header::RefDelta { 188 188 + base_id: id.to_owned(), 189 189 + }, 181 190 DeltaRef { object_index } => data::entry::Header::OfsDelta { 182 191 base_distance: index_to_base_distance(object_index), 183 192 },

+75 -20

knot2/third_party/gix-pack/src/find_traits.rs

View file Reviewed

··· 25 25 &self, 26 26 id: &gix_hash::oid, 27 27 buffer: &'a mut Vec<u8>, 28 28 - ) -> Result<Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, gix_object::find::Error> { 28 28 + ) -> Result< 29 29 + Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, 30 30 + gix_object::find::Error, 31 31 + > { 29 32 self.try_find_cached(id, buffer, &mut crate::cache::Never) 30 33 } 31 34 ··· 40 43 id: &gix_hash::oid, 41 44 buffer: &'a mut Vec<u8>, 42 45 pack_cache: &mut dyn crate::cache::DecodeEntry, 43 43 - ) -> Result<Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, gix_object::find::Error>; 46 46 + ) -> Result< 47 47 + Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, 48 48 + gix_object::find::Error, 49 49 + >; 44 50 45 51 /// Find the packs location where an object with `id` can be found in the database, or `None` if there is no pack 46 52 /// holding the object. 47 53 /// 48 54 /// _Note_ that this is always None if the object isn't packed even though it exists as loose object. 49 49 - fn location_by_oid(&self, id: &gix_hash::oid, buf: &mut Vec<u8>) -> Option<data::entry::Location>; 55 55 + fn location_by_oid( 56 56 + &self, 57 57 + id: &gix_hash::oid, 58 58 + buf: &mut Vec<u8>, 59 59 + ) -> Option<data::entry::Location>; 50 60 51 61 /// Obtain a vector of all offsets, in index order, along with their object id. 52 52 - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>>; 62 62 + fn pack_offsets_and_oid(&self, pack_id: u32) 63 63 + -> Option<Vec<(data::Offset, gix_hash::ObjectId)>>; 53 64 54 65 /// Return the [`find::Entry`] for `location` if it is backed by a pack. 55 66 /// ··· 64 75 } 65 76 66 77 mod ext { 67 67 - use gix_object::{BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TagRefIter, TreeRef, TreeRefIter}; 78 78 + use gix_object::{ 79 79 + BlobRef, CommitRef, CommitRefIter, Kind, ObjectRef, TagRef, TagRefIter, TreeRef, 80 80 + TreeRefIter, 81 81 + }; 68 82 69 83 macro_rules! make_obj_lookup { 70 84 ($method:ident, $object_variant:path, $object_kind:path, $object_type:ty) => { ··· 74 88 &self, 75 89 id: &gix_hash::oid, 76 90 buffer: &'a mut Vec<u8>, 77 77 - ) -> Result<($object_type, Option<crate::data::entry::Location>), gix_object::find::existing_object::Error> 78 78 - { 91 91 + ) -> Result< 92 92 + ($object_type, Option<crate::data::entry::Location>), 93 93 + gix_object::find::existing_object::Error, 94 94 + > { 79 95 let id = id.as_ref(); 80 96 self.try_find(id, buffer) 81 97 .map_err(gix_object::find::existing_object::Error::Find)? ··· 110 126 &self, 111 127 id: &gix_hash::oid, 112 128 buffer: &'a mut Vec<u8>, 113 113 - ) -> Result<($object_type, Option<crate::data::entry::Location>), gix_object::find::existing_iter::Error> { 129 129 + ) -> Result< 130 130 + ($object_type, Option<crate::data::entry::Location>), 131 131 + gix_object::find::existing_iter::Error, 132 132 + > { 114 133 let id = id.as_ref(); 115 134 self.try_find(id, buffer) 116 135 .map_err(gix_object::find::existing_iter::Error::Find)? ··· 137 156 &self, 138 157 id: &gix_hash::oid, 139 158 buffer: &'a mut Vec<u8>, 140 140 - ) -> Result<(gix_object::Data<'a>, Option<crate::data::entry::Location>), gix_object::find::existing::Error> 141 141 - { 159 159 + ) -> Result< 160 160 + (gix_object::Data<'a>, Option<crate::data::entry::Location>), 161 161 + gix_object::find::existing::Error, 162 162 + > { 142 163 self.try_find(id, buffer) 143 164 .map_err(gix_object::find::existing::Error::Find)? 144 165 .ok_or_else(|| gix_object::find::existing::Error::NotFound { ··· 150 171 make_obj_lookup!(find_tree, ObjectRef::Tree, Kind::Tree, TreeRef<'a>); 151 172 make_obj_lookup!(find_tag, ObjectRef::Tag, Kind::Tag, TagRef<'a>); 152 173 make_obj_lookup!(find_blob, ObjectRef::Blob, Kind::Blob, BlobRef<'a>); 153 153 - make_iter_lookup!(find_commit_iter, Kind::Blob, CommitRefIter<'a>, try_into_commit_iter); 154 154 - make_iter_lookup!(find_tree_iter, Kind::Tree, TreeRefIter<'a>, try_into_tree_iter); 174 174 + make_iter_lookup!( 175 175 + find_commit_iter, 176 176 + Kind::Blob, 177 177 + CommitRefIter<'a>, 178 178 + try_into_commit_iter 179 179 + ); 180 180 + make_iter_lookup!( 181 181 + find_tree_iter, 182 182 + Kind::Tree, 183 183 + TreeRefIter<'a>, 184 184 + try_into_tree_iter 185 185 + ); 155 186 make_iter_lookup!(find_tag_iter, Kind::Tag, TagRefIter<'a>, try_into_tag_iter); 156 187 } 157 188 ··· 179 210 id: &oid, 180 211 buffer: &'a mut Vec<u8>, 181 212 pack_cache: &mut dyn crate::cache::DecodeEntry, 182 182 - ) -> Result<Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, gix_object::find::Error> { 213 213 + ) -> Result< 214 214 + Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, 215 215 + gix_object::find::Error, 216 216 + > { 183 217 (*self).try_find_cached(id, buffer, pack_cache) 184 218 } 185 219 ··· 187 221 (*self).location_by_oid(id, buf) 188 222 } 189 223 190 190 - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 224 224 + fn pack_offsets_and_oid( 225 225 + &self, 226 226 + pack_id: u32, 227 227 + ) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 191 228 (*self).pack_offsets_and_oid(pack_id) 192 229 } 193 230 ··· 209 246 id: &oid, 210 247 buffer: &'a mut Vec<u8>, 211 248 pack_cache: &mut dyn crate::cache::DecodeEntry, 212 212 - ) -> Result<Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, gix_object::find::Error> { 249 249 + ) -> Result< 250 250 + Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, 251 251 + gix_object::find::Error, 252 252 + > { 213 253 self.deref().try_find_cached(id, buffer, pack_cache) 214 254 } 215 255 ··· 217 257 self.deref().location_by_oid(id, buf) 218 258 } 219 259 220 220 - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 260 260 + fn pack_offsets_and_oid( 261 261 + &self, 262 262 + pack_id: u32, 263 263 + ) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 221 264 self.deref().pack_offsets_and_oid(pack_id) 222 265 } 223 266 ··· 239 282 id: &oid, 240 283 buffer: &'a mut Vec<u8>, 241 284 pack_cache: &mut dyn crate::cache::DecodeEntry, 242 242 - ) -> Result<Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, gix_object::find::Error> { 285 285 + ) -> Result< 286 286 + Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, 287 287 + gix_object::find::Error, 288 288 + > { 243 289 self.deref().try_find_cached(id, buffer, pack_cache) 244 290 } 245 291 ··· 247 293 self.deref().location_by_oid(id, buf) 248 294 } 249 295 250 250 - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 296 296 + fn pack_offsets_and_oid( 297 297 + &self, 298 298 + pack_id: u32, 299 299 + ) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 251 300 self.deref().pack_offsets_and_oid(pack_id) 252 301 } 253 302 ··· 269 318 id: &oid, 270 319 buffer: &'a mut Vec<u8>, 271 320 pack_cache: &mut dyn crate::cache::DecodeEntry, 272 272 - ) -> Result<Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, gix_object::find::Error> { 321 321 + ) -> Result< 322 322 + Option<(gix_object::Data<'a>, Option<data::entry::Location>)>, 323 323 + gix_object::find::Error, 324 324 + > { 273 325 self.deref().try_find_cached(id, buffer, pack_cache) 274 326 } 275 327 ··· 277 329 self.deref().location_by_oid(id, buf) 278 330 } 279 331 280 280 - fn pack_offsets_and_oid(&self, pack_id: u32) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 332 332 + fn pack_offsets_and_oid( 333 333 + &self, 334 334 + pack_id: u32, 335 335 + ) -> Option<Vec<(data::Offset, gix_hash::ObjectId)>> { 281 336 self.deref().pack_offsets_and_oid(pack_id) 282 337 } 283 338

+24 -9

knot2/third_party/gix-pack/src/index/access.rs

View file Reviewed

··· 62 62 assert_eq!(oids.len(), crcs.len()); 63 63 assert_eq!(crcs.len(), offsets.len()); 64 64 match self.version { 65 65 - index::Version::V2 => izip!(oids, crcs, offsets).map(move |(oid, crc32, ofs32)| Entry { 66 66 - oid: gix_hash::ObjectId::from_bytes_or_panic(oid), 67 67 - pack_offset: self.pack_offset_from_offset_v2(ofs32, pack64_offset), 68 68 - crc32: Some(crate::read_u32(crc32)), 69 69 - }), 65 65 + index::Version::V2 => { 66 66 + izip!(oids, crcs, offsets).map(move |(oid, crc32, ofs32)| Entry { 67 67 + oid: gix_hash::ObjectId::from_bytes_or_panic(oid), 68 68 + pack_offset: self.pack_offset_from_offset_v2(ofs32, pack64_offset), 69 69 + crc32: Some(crate::read_u32(crc32)), 70 70 + }) 71 71 + } 70 72 _ => panic!("Cannot use iter_v2() on index of type {:?}", self.version), 71 73 } 72 74 } ··· 96 98 match self.version { 97 99 index::Version::V2 => { 98 100 let start = self.offset_pack_offset_v2() + index * N32_SIZE; 99 99 - self.pack_offset_from_offset_v2(&self.data[start..][..N32_SIZE], self.offset_pack_offset64_v2()) 101 101 + self.pack_offset_from_offset_v2( 102 102 + &self.data[start..][..N32_SIZE], 103 103 + self.offset_pack_offset64_v2(), 104 104 + ) 100 105 } 101 106 index::Version::V1 => { 102 107 let start = V1_HEADER_SIZE + index * (N32_SIZE + self.hash_len); ··· 170 175 index::Version::V1 => self.iter().map(|e| e.pack_offset).collect(), 171 176 index::Version::V2 => { 172 177 let offset32_start = &self.data[self.offset_pack_offset_v2()..]; 173 173 - let offsets32 = offset32_start.chunks_exact(N32_SIZE).take(self.num_objects as usize); 178 178 + let offsets32 = offset32_start 179 179 + .chunks_exact(N32_SIZE) 180 180 + .take(self.num_objects as usize); 174 181 assert_eq!(self.num_objects as usize, offsets32.len()); 175 182 let pack_offset_64_start = self.offset_pack_offset64_v2(); 176 183 offsets32 ··· 219 226 ) -> Option<PrefixLookupResult> { 220 227 let first_byte = prefix.as_oid().first_byte() as usize; 221 228 let mut upper_bound = fan[first_byte]; 222 222 - let mut lower_bound = if first_byte != 0 { fan[first_byte - 1] } else { 0 }; 229 229 + let mut lower_bound = if first_byte != 0 { 230 230 + fan[first_byte - 1] 231 231 + } else { 232 232 + 0 233 233 + }; 223 234 224 235 // Bisect using indices 225 236 while lower_bound < upper_bound { ··· 280 291 ) -> Option<EntryIndex> { 281 292 let first_byte = id.first_byte() as usize; 282 293 let mut upper_bound = fan[first_byte]; 283 283 - let mut lower_bound = if first_byte != 0 { fan[first_byte - 1] } else { 0 }; 294 294 + let mut lower_bound = if first_byte != 0 { 295 295 + fan[first_byte - 1] 296 296 + } else { 297 297 + 0 298 298 + }; 284 299 285 300 while lower_bound < upper_bound { 286 301 let mid = u32::midpoint(lower_bound, upper_bound);

+21 -7

knot2/third_party/gix-pack/src/index/encode.rs

View file Reviewed

··· 13 13 for (offset_be, byte) in fan_out.iter_mut().zip(0u8..=255) { 14 14 *offset_be = match idx_and_entry.as_ref() { 15 15 Some((_idx, first_byte)) => match first_byte.cmp(&byte) { 16 16 - Ordering::Less => unreachable!("ids should be ordered, and we make sure to keep ahead with them"), 16 16 + Ordering::Less => { 17 17 + unreachable!("ids should be ordered, and we make sure to keep ahead with them") 18 18 + } 17 19 Ordering::Greater => upper_bound, 18 20 Ordering::Equal => { 19 21 if byte == 255 { 20 22 entries_len 21 23 } else { 22 24 idx_and_entry = iter.find(|(_, first_byte)| *first_byte != byte); 23 23 - upper_bound = idx_and_entry.as_ref().map_or(entries_len, |(idx, _)| *idx as u32); 25 25 + upper_bound = idx_and_entry 26 26 + .as_ref() 27 27 + .map_or(entries_len, |(idx, _)| *idx as u32); 24 28 upper_bound 25 29 } 26 30 } ··· 76 80 progress: &mut dyn DynNestedProgress, 77 81 ) -> Result<gix_hash::ObjectId, gix_hash::io::Error> { 78 82 use io::Write; 79 79 - assert_eq!(kind, crate::index::Version::V2, "Can only write V2 packs right now"); 83 83 + assert_eq!( 84 84 + kind, 85 85 + crate::index::Version::V2, 86 86 + "Can only write V2 packs right now" 87 87 + ); 80 88 assert!( 81 89 entries_sorted_by_oid.len() <= u32::MAX as usize, 82 90 "a pack cannot have more than u32::MAX objects" ··· 92 100 93 101 progress.init(Some(4), progress::steps()); 94 102 let start = std::time::Instant::now(); 95 95 - let _info = progress.add_child_with_id("writing fan-out table".into(), gix_features::progress::UNKNOWN); 103 103 + let _info = progress.add_child_with_id( 104 104 + "writing fan-out table".into(), 105 105 + gix_features::progress::UNKNOWN, 106 106 + ); 96 107 let fan_out = fanout(&mut entries_sorted_by_oid.iter().map(|e| e.data.id.first_byte())); 97 108 98 109 for value in fan_out.iter() { ··· 100 111 } 101 112 102 113 progress.inc(); 103 103 - let _info = progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN); 114 114 + let _info = 115 115 + progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN); 104 116 for entry in &entries_sorted_by_oid { 105 117 out.write_all(entry.data.id.as_slice())?; 106 118 } 107 119 108 120 progress.inc(); 109 109 - let _info = progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN); 121 121 + let _info = 122 122 + progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN); 110 123 for entry in &entries_sorted_by_oid { 111 124 out.write_all(&entry.data.crc32.to_be_bytes())?; 112 125 } 113 126 114 127 progress.inc(); 115 115 - let _info = progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN); 128 128 + let _info = 129 129 + progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN); 116 130 { 117 131 let mut offsets64 = Vec::<u64>::new(); 118 132 for entry in &entries_sorted_by_oid {

+25 -11

knot2/third_party/gix-pack/src/index/init.rs

View file Reviewed

··· 53 53 let footer_size = hash_len * 2; 54 54 if idx_len < FAN_LEN * N32_SIZE + footer_size { 55 55 return Err(Error::Corrupt { 56 56 - message: format!("Pack index of size {idx_len} is too small for even an empty index"), 56 56 + message: format!( 57 57 + "Pack index of size {idx_len} is too small for even an empty index" 58 58 + ), 57 59 }); 58 60 } 59 61 let (kind, fan, num_objects) = { ··· 116 118 Ok(()) 117 119 } 118 120 119 119 - fn validate_size(data: &[u8], kind: Version, num_objects: u32, hash_len: usize) -> Result<(), Error> { 121 121 + fn validate_size( 122 122 + data: &[u8], 123 123 + kind: Version, 124 124 + num_objects: u32, 125 125 + hash_len: usize, 126 126 + ) -> Result<(), Error> { 120 127 let num_objects = num_objects as usize; 121 128 let footer_size = hash_len * 2; 122 129 let expected_size = match kind { ··· 129 136 })?, 130 137 Version::V2 => { 131 138 let v2_header_size = V2_SIGNATURE.len() + N32_SIZE + FAN_LEN * N32_SIZE; 132 132 - let oid_bytes = num_objects.checked_mul(hash_len).ok_or_else(|| Error::Corrupt { 133 133 - message: "Pack index size overflowed while validating object ids".into(), 134 134 - })?; 135 135 - let table_bytes = num_objects.checked_mul(N32_SIZE).ok_or_else(|| Error::Corrupt { 136 136 - message: "Pack index size overflowed while validating 32-bit tables".into(), 137 137 - })?; 139 139 + let oid_bytes = num_objects 140 140 + .checked_mul(hash_len) 141 141 + .ok_or_else(|| Error::Corrupt { 142 142 + message: "Pack index size overflowed while validating object ids".into(), 143 143 + })?; 144 144 + let table_bytes = num_objects 145 145 + .checked_mul(N32_SIZE) 146 146 + .ok_or_else(|| Error::Corrupt { 147 147 + message: "Pack index size overflowed while validating 32-bit tables".into(), 148 148 + })?; 138 149 let offset32_start = v2_header_size 139 150 .checked_add(oid_bytes) 140 151 .and_then(|size| size.checked_add(table_bytes)) 141 152 .ok_or_else(|| Error::Corrupt { 142 153 message: "Pack index size overflowed while locating 32-bit offsets".into(), 143 154 })?; 144 144 - let offset32_end = offset32_start.checked_add(table_bytes).ok_or_else(|| Error::Corrupt { 145 145 - message: "Pack index size overflowed while locating 32-bit offsets".into(), 146 146 - })?; 155 155 + let offset32_end = 156 156 + offset32_start 157 157 + .checked_add(table_bytes) 158 158 + .ok_or_else(|| Error::Corrupt { 159 159 + message: "Pack index size overflowed while locating 32-bit offsets".into(), 160 160 + })?; 147 161 if offset32_end > data.len() { 148 162 return Err(Error::Corrupt { 149 163 message: format!(

+13 -3

knot2/third_party/gix-pack/src/index/traverse/mod.rs

View file Reviewed

··· 92 92 where 93 93 C: crate::cache::DecodeEntry, 94 94 E: std::error::Error + Send + Sync + 'static, 95 95 - Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + Send + Clone, 95 95 + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> 96 96 + + Send 97 97 + + Clone, 96 98 F: Fn() -> C + Send + Clone, 97 99 { 98 100 match traversal { ··· 112 114 processor, 113 115 progress, 114 116 should_interrupt, 115 115 - with_index::Options { check, thread_limit }, 117 117 + with_index::Options { 118 118 + check, 119 119 + thread_limit, 120 120 + }, 116 121 ), 117 122 } 118 123 } ··· 153 158 inflate: &mut zlib::Inflate, 154 159 progress: &mut dyn Progress, 155 160 index_entry: &index::Entry, 156 156 - processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E>, 161 161 + processor: &mut impl FnMut( 162 162 + gix_object::Kind, 163 163 + &[u8], 164 164 + &index::Entry, 165 165 + &dyn Progress, 166 166 + ) -> Result<(), E>, 157 167 ) -> Result<crate::data::decode::entry::Outcome, Error<E>> 158 168 where 159 169 C: crate::cache::DecodeEntry,

+8 -2

knot2/third_party/gix-pack/src/index/traverse/reduce.rs

View file Reviewed

··· 86 86 let chunk_total = chunk_stats.into_iter().fold( 87 87 data::decode::entry::Outcome::default_from_kind(gix_object::Kind::Tree), 88 88 |mut total, stats| { 89 89 - *self.stats.objects_per_chain_length.entry(stats.num_deltas).or_insert(0) += 1; 89 89 + *self 90 90 + .stats 91 91 + .objects_per_chain_length 92 92 + .entry(stats.num_deltas) 93 93 + .or_insert(0) += 1; 90 94 self.stats.total_decompressed_entries_size += stats.decompressed_size; 91 95 self.stats.total_compressed_entries_size += stats.compressed_size as u64; 92 96 self.stats.total_object_size += stats.object_size; ··· 122 126 self.entries_seen, 123 127 elapsed_s, 124 128 objects_per_second, 125 125 - gix_features::progress::bytesize::ByteSize(self.stats.average.object_size * u64::from(objects_per_second)) 129 129 + gix_features::progress::bytesize::ByteSize( 130 130 + self.stats.average.object_size * u64::from(objects_per_second) 131 131 + ) 126 132 )); 127 133 Ok(self.stats) 128 134 }

+4 -1

knot2/third_party/gix-pack/src/index/traverse/types.rs

View file Reviewed

··· 72 72 matches!(self, SafetyCheck::All) 73 73 } 74 74 pub(crate) fn object_checksum(&self) -> bool { 75 75 - matches!(self, SafetyCheck::All | SafetyCheck::SkipFileChecksumVerification) 75 75 + matches!( 76 76 + self, 77 77 + SafetyCheck::All | SafetyCheck::SkipFileChecksumVerification 78 78 + ) 76 79 } 77 80 pub(crate) fn fatal_decode_error(&self) -> bool { 78 81 match self {

+11 -7

knot2/third_party/gix-pack/src/index/traverse/with_index.rs

View file Reviewed

··· 126 126 self.object_hash, 127 127 )?; 128 128 let mut outcome = digest_statistics(tree.traverse( 129 129 - |slice: crate::data::EntryRange, source: &crate::data::File, buf: &mut Vec<u8>| { 130 130 - source.read_into(slice, buf) 131 131 - }, 129 129 + |slice: crate::data::EntryRange, 130 130 + source: &crate::data::File, 131 131 + buf: &mut Vec<u8>| { source.read_into(slice, buf) }, 132 132 pack, 133 133 pack.pack_end() as u64, 134 134 move |data, ··· 171 171 } 172 172 }, 173 173 traverse::Options { 174 174 - object_progress: Box::new( 175 175 - progress.add_child_with_id("Resolving".into(), ProgressId::DecodedObjects.into()), 176 176 - ), 174 174 + object_progress: Box::new(progress.add_child_with_id( 175 175 + "Resolving".into(), 176 176 + ProgressId::DecodedObjects.into(), 177 177 + )), 177 178 size_progress: 178 178 - &mut progress.add_child_with_id("Decoding".into(), ProgressId::DecodedBytes.into()), 179 179 + &mut progress.add_child_with_id( 180 180 + "Decoding".into(), 181 181 + ProgressId::DecodedBytes.into(), 182 182 + ), 179 183 thread_limit, 180 184 should_interrupt, 181 185 object_hash: self.object_hash,

+28 -9

knot2/third_party/gix-pack/src/index/traverse/with_lookup.rs

View file Reviewed

··· 84 84 where 85 85 C: crate::cache::DecodeEntry, 86 86 E: std::error::Error + Send + Sync + 'static, 87 87 - Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> + Send + Clone, 87 87 + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn Progress) -> Result<(), E> 88 88 + + Send 89 89 + + Clone, 88 90 F: Fn() -> C + Send + Clone, 89 91 { 90 92 let (verify_result, traversal_result) = parallel::join( ··· 98 100 ProgressId::HashPackIndexBytes.into(), 99 101 ); 100 102 move || { 101 101 - let res = 102 102 - self.possibly_verify(pack, check, &mut pack_progress, &mut index_progress, should_interrupt); 103 103 + let res = self.possibly_verify( 104 104 + pack, 105 105 + check, 106 106 + &mut pack_progress, 107 107 + &mut index_progress, 108 108 + should_interrupt, 109 109 + ); 103 110 if res.is_err() { 104 111 should_interrupt.store(true, Ordering::SeqCst); 105 112 } ··· 116 123 ); 117 124 118 125 let (chunk_size, thread_limit, available_cores) = 119 119 - parallel::optimize_chunk_size_and_thread_limit(1000, Some(index_entries.len()), thread_limit, None); 120 120 - let there_are_enough_entries_to_process = || index_entries.len() > chunk_size * available_cores; 126 126 + parallel::optimize_chunk_size_and_thread_limit( 127 127 + 1000, 128 128 + Some(index_entries.len()), 129 129 + thread_limit, 130 130 + None, 131 131 + ); 132 132 + let there_are_enough_entries_to_process = 133 133 + || index_entries.len() > chunk_size * available_cores; 121 134 let input_chunks = index_entries.chunks(chunk_size); 122 135 let reduce_progress = OwnShared::new(Mutable::new({ 123 123 - let mut p = progress.add_child_with_id("Traversing".into(), ProgressId::DecodedObjects.into()); 124 124 - p.init(Some(self.num_objects() as usize), progress::count("objects")); 136 136 + let mut p = progress 137 137 + .add_child_with_id("Traversing".into(), ProgressId::DecodedObjects.into()); 138 138 + p.init( 139 139 + Some(self.num_objects() as usize), 140 140 + progress::count("objects"), 141 141 + ); 125 142 p 126 143 })); 127 144 let state_per_thread = { ··· 131 148 make_pack_lookup_cache(), 132 149 Vec::with_capacity(2048), // decode buffer 133 150 zlib::Inflate::default(), 134 134 - lock(&reduce_progress) 135 135 - .add_child_with_id(format!("thread {index}"), gix_features::progress::UNKNOWN), // per thread progress 151 151 + lock(&reduce_progress).add_child_with_id( 152 152 + format!("thread {index}"), 153 153 + gix_features::progress::UNKNOWN, 154 154 + ), // per thread progress 136 155 ) 137 156 } 138 157 };

+21 -9

knot2/third_party/gix-pack/src/index/verify.rs

View file Reviewed

··· 17 17 pub enum Error { 18 18 #[error("Reserialization of an object failed")] 19 19 Io(#[from] std::io::Error), 20 20 - #[error("The fan at index {index} is out of order as it's larger then the following value.")] 20 20 + #[error( 21 21 + "The fan at index {index} is out of order as it's larger then the following value." 22 22 + )] 21 23 Fan { index: usize }, 22 24 #[error("{kind} object {id} could not be decoded")] 23 25 ObjectDecode { ··· 25 27 kind: gix_object::Kind, 26 28 id: gix_hash::ObjectId, 27 29 }, 28 28 - #[error("{kind} object {id} wasn't re-encoded without change, wanted\n{expected}\n\nGOT\n\n{actual}")] 30 30 + #[error( 31 31 + "{kind} object {id} wasn't re-encoded without change, wanted\n{expected}\n\nGOT\n\n{actual}" 32 32 + )] 29 33 ObjectEncodeMismatch { 30 34 kind: gix_object::Kind, 31 35 id: gix_hash::ObjectId, ··· 203 207 { 204 208 let mut encode_buf = Vec::with_capacity(2048); 205 209 move |kind, data, index_entry, progress| { 206 206 - Self::verify_entry(verify_mode, &mut encode_buf, kind, data, index_entry, progress) 210 210 + Self::verify_entry( 211 211 + verify_mode, 212 212 + &mut encode_buf, 213 213 + kind, 214 214 + data, 215 215 + index_entry, 216 216 + progress, 217 217 + ) 207 218 } 208 219 }, 209 220 index::traverse::Options { ··· 219 230 }), 220 231 None => self 221 232 .verify_checksum( 222 222 - &mut progress 223 223 - .add_child_with_id("Sha1 of index".into(), integrity::ProgressId::ChecksumBytes.into()), 233 233 + &mut progress.add_child_with_id( 234 234 + "Sha1 of index".into(), 235 235 + integrity::ProgressId::ChecksumBytes.into(), 236 236 + ), 224 237 should_interrupt, 225 238 ) 226 239 .map_err(index::traverse::Error::IndexVerify) ··· 245 258 match object_kind { 246 259 Tree | Commit | Tag => { 247 260 let object = 248 248 - gix_object::ObjectRef::from_bytes(buf, object_kind, index_entry.oid.kind()).map_err(|err| { 249 249 - integrity::Error::ObjectDecode { 261 261 + gix_object::ObjectRef::from_bytes(buf, object_kind, index_entry.oid.kind()) 262 262 + .map_err(|err| integrity::Error::ObjectDecode { 250 263 source: err, 251 264 kind: object_kind, 252 265 id: index_entry.oid, 253 253 - } 254 254 - })?; 266 266 + })?; 255 267 if let Mode::HashCrc32DecodeEncode = verify_mode { 256 268 encode_buf.clear(); 257 269 object.write_to(&mut *encode_buf)?;

+9 -3

knot2/third_party/gix-pack/src/multi_index/chunk.rs

View file Reviewed

··· 66 66 out.try_reserve(num_packs)?; 67 67 68 68 for _ in 0..num_packs { 69 69 - let null_byte_pos = chunk.find_byte(b'\0').ok_or(decode::Error::MissingNullByte)?; 69 69 + let null_byte_pos = chunk 70 70 + .find_byte(b'\0') 71 71 + .ok_or(decode::Error::MissingNullByte)?; 70 72 71 73 let path = &chunk[..null_byte_pos]; 72 74 if alloc_limit_bytes.is_some_and(|limit| path.len() > limit) { ··· 168 170 sorted_entries: &[multi_index::write::Entry], 169 171 out: &mut dyn std::io::Write, 170 172 ) -> std::io::Result<()> { 171 171 - let fanout = crate::index::encode::fanout(&mut sorted_entries.iter().map(|e| e.id.first_byte())); 173 173 + let fanout = 174 174 + crate::index::encode::fanout(&mut sorted_entries.iter().map(|e| e.id.first_byte())); 172 175 173 176 for value in fanout.iter() { 174 177 out.write_all(&value.to_be_bytes())?; ··· 301 304 .checked_sub(1) 302 305 .expect("BUG: wrote more offsets the previously found"); 303 306 } 304 304 - assert_eq!(num_large_offsets, 0, "BUG: wrote less offsets than initially counted"); 307 307 + assert_eq!( 308 308 + num_large_offsets, 0, 309 309 + "BUG: wrote less offsets than initially counted" 310 310 + ); 305 311 Ok(()) 306 312 } 307 313

+16 -5

knot2/third_party/gix-pack/src/multi_index/init.rs

View file Reviewed

··· 29 29 #[error(transparent)] 30 30 PackNames(#[from] chunk::index_names::decode::Error), 31 31 #[error("multi-index chunk {:?} has invalid size: {message}", String::from_utf8_lossy(.id))] 32 32 - InvalidChunkSize { id: gix_chunk::Id, message: &'static str }, 32 32 + InvalidChunkSize { 33 33 + id: gix_chunk::Id, 34 34 + message: &'static str, 35 35 + }, 33 36 } 34 37 } 35 38 ··· 65 68 /// 66 69 /// It is used to reject reserving the output `Vec<PathBuf>` if its capacity estimate exceeds the limit, 67 70 /// and to reject any single path entry whose byte length exceeds the limit before turning it into a `PathBuf`. 68 68 - pub fn from_data(data: T, path: PathBuf, alloc_limit_bytes: Option<usize>) -> Result<Self, Error> { 71 71 + pub fn from_data( 72 72 + data: T, 73 73 + path: PathBuf, 74 74 + alloc_limit_bytes: Option<usize>, 75 75 + ) -> Result<Self, Error> { 69 76 const TRAILER_LEN: usize = gix_hash::Kind::shortest().len_in_bytes(); /* trailing hash */ 70 77 if data.len() 71 78 < Self::HEADER_LEN 72 72 - + gix_chunk::file::Index::size_for_entries(4 /*index names, fan, offsets, oids*/) 79 79 + + gix_chunk::file::Index::size_for_entries( 80 80 + 4, /*index names, fan, offsets, oids*/ 81 81 + ) 73 82 + chunk::fanout::SIZE 74 83 + TRAILER_LEN 75 84 { ··· 105 114 (version, object_hash, num_chunks, num_indices) 106 115 }; 107 116 108 108 - let chunks = gix_chunk::file::Index::from_bytes(&data, Self::HEADER_LEN, u32::from(num_chunks))?; 117 117 + let chunks = 118 118 + gix_chunk::file::Index::from_bytes(&data, Self::HEADER_LEN, u32::from(num_chunks))?; 109 119 110 120 let index_names = chunks.data_by_id(&data, chunk::index_names::ID)?; 111 111 - let index_names = chunk::index_names::from_bytes(index_names, num_indices, alloc_limit_bytes)?; 121 121 + let index_names = 122 122 + chunk::index_names::from_bytes(index_names, num_indices, alloc_limit_bytes)?; 112 123 113 124 let fan = chunks.data_by_id(&data, chunk::fanout::ID)?; 114 125 let fan = chunk::fanout::from_bytes(fan).ok_or(Error::MultiPackFanSize)?;

+23 -10

knot2/third_party/gix-pack/src/multi_index/verify.rs

View file Reviewed

··· 12 12 #[derive(thiserror::Error, Debug)] 13 13 #[allow(missing_docs)] 14 14 pub enum Error { 15 15 - #[error("Object {id} should be at pack-offset {expected_pack_offset} but was found at {actual_pack_offset}")] 15 15 + #[error( 16 16 + "Object {id} should be at pack-offset {expected_pack_offset} but was found at {actual_pack_offset}" 17 17 + )] 16 18 PackOffsetMismatch { 17 19 id: gix_hash::ObjectId, 18 20 expected_pack_offset: u64, ··· 30 32 OidNotFound { id: gix_hash::ObjectId }, 31 33 #[error("The object id at multi-index entry {index} wasn't in order")] 32 34 OutOfOrder { index: EntryIndex }, 33 33 - #[error("The fan at index {index} is out of order as it's larger then the following value.")] 35 35 + #[error( 36 36 + "The fan at index {index} is out of order as it's larger then the following value." 37 37 + )] 34 38 Fan { index: usize }, 35 39 #[error("The multi-index claims to have no objects")] 36 40 Empty, ··· 178 182 let mut pack_ids_and_offsets = exact_vec(self.num_objects as usize); 179 183 { 180 184 let order_start = Instant::now(); 181 181 - let mut progress = progress.add_child_with_id("checking oid order".into(), gix_features::progress::UNKNOWN); 185 185 + let mut progress = progress 186 186 + .add_child_with_id("checking oid order".into(), gix_features::progress::UNKNOWN); 182 187 progress.init( 183 188 Some(self.num_objects as usize), 184 189 gix_features::progress::count("objects"), ··· 189 194 let rhs = self.oid_at_index(entry_index + 1); 190 195 191 196 if rhs.cmp(lhs) != Ordering::Greater { 192 192 - return Err(index::traverse::Error::Processor(integrity::Error::OutOfOrder { 193 193 - index: entry_index, 194 194 - })); 197 197 + return Err(index::traverse::Error::Processor( 198 198 + integrity::Error::OutOfOrder { index: entry_index }, 199 199 + )); 195 200 } 196 201 let (pack_id, _) = self.pack_id_and_pack_offset_at_index(entry_index); 197 202 pack_ids_and_offsets.push((pack_id, entry_index)); ··· 230 235 } else { 231 236 index = Some( 232 237 index::File::at(index_path, self.object_hash) 233 233 - .map_err(|err| integrity::Error::BundleInit(crate::bundle::init::Error::Index(err))) 238 238 + .map_err(|err| { 239 239 + integrity::Error::BundleInit(crate::bundle::init::Error::Index(err)) 240 240 + }) 234 241 .map_err(index::traverse::Error::Processor)?, 235 242 ); 236 243 index.as_ref().expect("just set") ··· 254 261 let oid = self.oid_at_index(entry_id); 255 262 let (_, expected_pack_offset) = self.pack_id_and_pack_offset_at_index(entry_id); 256 263 let entry_in_bundle_index = index.lookup(oid).ok_or_else(|| { 257 257 - index::traverse::Error::Processor(integrity::Error::OidNotFound { id: oid.to_owned() }) 264 264 + index::traverse::Error::Processor(integrity::Error::OidNotFound { 265 265 + id: oid.to_owned(), 266 266 + }) 258 267 })?; 259 268 let actual_pack_offset = index.pack_offset_at_index(entry_in_bundle_index); 260 269 if actual_pack_offset != expected_pack_offset { ··· 270 279 } 271 280 272 281 if should_interrupt.load(std::sync::atomic::Ordering::Relaxed) { 273 273 - return Err(index::traverse::Error::Processor(integrity::Error::Interrupted)); 282 282 + return Err(index::traverse::Error::Processor( 283 283 + integrity::Error::Interrupted, 284 284 + )); 274 285 } 275 286 offsets_progress.show_throughput(offset_start); 276 287 } ··· 295 306 PackDecode { id, offset, source } => PackDecode { id, offset, source }, 296 307 PackMismatch(err) => PackMismatch(err), 297 308 EntryType(err) => EntryType(err), 298 298 - PackObjectVerify { offset, source } => PackObjectVerify { offset, source }, 309 309 + PackObjectVerify { offset, source } => { 310 310 + PackObjectVerify { offset, source } 311 311 + } 299 312 Crc32Mismatch { 300 313 expected, 301 314 actual,

+52 -18

knot2/third_party/gix-pack/src/multi_index/write.rs

View file Reviewed

··· 108 108 "Collecting entries".into(), 109 109 ProgressId::FromPathsCollectingEntries.into(), 110 110 ); 111 111 - progress.init(Some(index_paths_sorted.len()), gix_features::progress::count("indices")); 111 111 + progress.init( 112 112 + Some(index_paths_sorted.len()), 113 113 + gix_features::progress::count("indices"), 114 114 + ); 112 115 113 116 // This could be parallelized… but it's probably not worth it unless you have 500mio objects. 114 117 for (index_id, index) in index_paths_sorted.iter().enumerate() { ··· 134 137 135 138 let start = Instant::now(); 136 139 progress.set_name("Deduplicate".into()); 137 137 - progress.init(Some(entries.len()), gix_features::progress::count("entries")); 140 140 + progress.init( 141 141 + Some(entries.len()), 142 142 + gix_features::progress::count("entries"), 143 143 + ); 138 144 entries.sort_by(|l, r| { 139 145 l.id.cmp(&r.id) 140 146 .then_with(|| l.index_mtime.cmp(&r.index_mtime).reverse()) ··· 154 160 multi_index::chunk::index_names::ID, 155 161 multi_index::chunk::index_names::storage_size(&index_filenames_sorted), 156 162 ); 157 157 - cf.plan_chunk(multi_index::chunk::fanout::ID, multi_index::chunk::fanout::SIZE as u64); 163 163 + cf.plan_chunk( 164 164 + multi_index::chunk::fanout::ID, 165 165 + multi_index::chunk::fanout::SIZE as u64, 166 166 + ); 158 167 cf.plan_chunk( 159 168 multi_index::chunk::lookup::ID, 160 169 multi_index::chunk::lookup::storage_size(entries.len(), object_hash), ··· 172 181 ); 173 182 } 174 183 175 175 - let mut write_progress = 176 176 - progress.add_child_with_id("Writing multi-index".into(), ProgressId::BytesWritten.into()); 184 184 + let mut write_progress = progress.add_child_with_id( 185 185 + "Writing multi-index".into(), 186 186 + ProgressId::BytesWritten.into(), 187 187 + ); 177 188 let write_start = Instant::now(); 178 189 write_progress.init( 179 190 Some(cf.planned_storage_size() as usize + multi_index::File::<MMap>::HEADER_LEN), ··· 186 197 187 198 let bytes_written = multi_index::File::<MMap>::write_header( 188 199 &mut out, 189 189 - cf.num_chunks().try_into().expect("BUG: wrote more than 256 chunks"), 200 200 + cf.num_chunks() 201 201 + .try_into() 202 202 + .expect("BUG: wrote more than 256 chunks"), 190 203 index_paths_sorted.len() as u32, 191 204 object_hash, 192 205 ) ··· 194 207 195 208 { 196 209 progress.set_name("Writing chunks".into()); 197 197 - progress.init(Some(cf.num_chunks()), gix_features::progress::count("chunks")); 210 210 + progress.init( 211 211 + Some(cf.num_chunks()), 212 212 + gix_features::progress::count("chunks"), 213 213 + ); 198 214 199 215 let mut chunk_write = cf 200 216 .into_write(&mut out, bytes_written) 201 217 .map_err(gix_hash::io::Error::from)?; 202 218 while let Some(chunk_to_write) = chunk_write.next_chunk() { 203 219 match chunk_to_write { 204 204 - multi_index::chunk::index_names::ID => { 205 205 - multi_index::chunk::index_names::write(&index_filenames_sorted, &mut chunk_write) 220 220 + multi_index::chunk::index_names::ID => multi_index::chunk::index_names::write( 221 221 + &index_filenames_sorted, 222 222 + &mut chunk_write, 223 223 + ), 224 224 + multi_index::chunk::fanout::ID => { 225 225 + multi_index::chunk::fanout::write(&entries, &mut chunk_write) 206 226 } 207 207 - multi_index::chunk::fanout::ID => multi_index::chunk::fanout::write(&entries, &mut chunk_write), 208 208 - multi_index::chunk::lookup::ID => multi_index::chunk::lookup::write(&entries, &mut chunk_write), 209 209 - multi_index::chunk::offsets::ID => { 210 210 - multi_index::chunk::offsets::write(&entries, num_large_offsets.is_some(), &mut chunk_write) 227 227 + multi_index::chunk::lookup::ID => { 228 228 + multi_index::chunk::lookup::write(&entries, &mut chunk_write) 211 229 } 212 212 - multi_index::chunk::large_offsets::ID => multi_index::chunk::large_offsets::write( 230 230 + multi_index::chunk::offsets::ID => multi_index::chunk::offsets::write( 213 231 &entries, 214 214 - num_large_offsets.expect("available if planned"), 232 232 + num_large_offsets.is_some(), 215 233 &mut chunk_write, 216 234 ), 217 217 - unknown => unreachable!("BUG: forgot to implement chunk {:?}", std::str::from_utf8(&unknown)), 235 235 + multi_index::chunk::large_offsets::ID => { 236 236 + multi_index::chunk::large_offsets::write( 237 237 + &entries, 238 238 + num_large_offsets.expect("available if planned"), 239 239 + &mut chunk_write, 240 240 + ) 241 241 + } 242 242 + unknown => unreachable!( 243 243 + "BUG: forgot to implement chunk {:?}", 244 244 + std::str::from_utf8(&unknown) 245 245 + ), 218 246 } 219 247 .map_err(gix_hash::io::Error::from)?; 220 248 progress.inc(); ··· 225 253 } 226 254 227 255 // write trailing checksum 228 228 - let multi_index_checksum = out.inner.hash.try_finalize().map_err(gix_hash::io::Error::from)?; 256 256 + let multi_index_checksum = out 257 257 + .inner 258 258 + .hash 259 259 + .try_finalize() 260 260 + .map_err(gix_hash::io::Error::from)?; 229 261 out.inner 230 262 .inner 231 263 .write_all(multi_index_checksum.as_slice()) 232 264 .map_err(gix_hash::io::Error::from)?; 233 265 out.progress.show_throughput(write_start); 234 266 235 235 - Ok(Outcome { multi_index_checksum }) 267 267 + Ok(Outcome { 268 268 + multi_index_checksum, 269 269 + }) 236 270 } 237 271 } 238 272