alice.pds.demo.boltless.dev
This repository has no description
did:plc:3fwecdnvtcscjnrx2p4n7a…
1use std::cmp::Ordering;
2
3pub(crate) const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff;
4pub(crate) const HIGH_BIT: u32 = 0x8000_0000;
5
6pub(crate) fn fanout(iter: &mut dyn ExactSizeIterator<Item = u8>) -> [u32; 256] {
7 let mut fan_out = [0u32; 256];
8 let entries_len = iter.len() as u32;
9 let mut iter = iter.enumerate();
10 let mut idx_and_entry = iter.next();
11 let mut upper_bound = 0;
12
13 for (offset_be, byte) in fan_out.iter_mut().zip(0u8..=255) {
14 *offset_be = match idx_and_entry.as_ref() {
15 Some((_idx, first_byte)) => match first_byte.cmp(&byte) {
16 Ordering::Less => unreachable!("ids should be ordered, and we make sure to keep ahead with them"),
17 Ordering::Greater => upper_bound,
18 Ordering::Equal => {
19 if byte == 255 {
20 entries_len
21 } else {
22 idx_and_entry = iter.find(|(_, first_byte)| *first_byte != byte);
23 upper_bound = idx_and_entry.as_ref().map_or(entries_len, |(idx, _)| *idx as u32);
24 upper_bound
25 }
26 }
27 },
28 None => entries_len,
29 };
30 }
31
32 fan_out
33}
34
35#[cfg(feature = "streaming-input")]
36mod function {
37 use std::io;
38
39 use gix_features::progress::{self, DynNestedProgress};
40
41 use super::{HIGH_BIT, LARGE_OFFSET_THRESHOLD, fanout};
42 use crate::index::V2_SIGNATURE;
43
44 struct Count<W> {
45 bytes: u64,
46 inner: W,
47 }
48
49 impl<W> Count<W> {
50 fn new(inner: W) -> Self {
51 Count { bytes: 0, inner }
52 }
53 }
54
55 impl<W> io::Write for Count<W>
56 where
57 W: io::Write,
58 {
59 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
60 let written = self.inner.write(buf)?;
61 self.bytes += written as u64;
62 Ok(written)
63 }
64
65 fn flush(&mut self) -> io::Result<()> {
66 self.inner.flush()
67 }
68 }
69
70 pub(crate) fn write_to(
71 out: &mut dyn io::Write,
72 entries_sorted_by_oid: Vec<crate::cache::delta::Item<crate::index::write::TreeEntry>>,
73 pack_hash: &gix_hash::ObjectId,
74 kind: crate::index::Version,
75 object_hash: gix_hash::Kind,
76 progress: &mut dyn DynNestedProgress,
77 ) -> Result<gix_hash::ObjectId, gix_hash::io::Error> {
78 use io::Write;
79 assert_eq!(kind, crate::index::Version::V2, "Can only write V2 packs right now");
80 assert!(
81 entries_sorted_by_oid.len() <= u32::MAX as usize,
82 "a pack cannot have more than u32::MAX objects"
83 );
84
85 // Write header
86 let mut out = Count::new(std::io::BufWriter::with_capacity(
87 8 * 4096,
88 gix_hash::io::Write::new(out, object_hash),
89 ));
90 out.write_all(V2_SIGNATURE)?;
91 out.write_all(&(kind as u32).to_be_bytes())?;
92
93 progress.init(Some(4), progress::steps());
94 let start = std::time::Instant::now();
95 let _info = progress.add_child_with_id("writing fan-out table".into(), gix_features::progress::UNKNOWN);
96 let fan_out = fanout(&mut entries_sorted_by_oid.iter().map(|e| e.data.id.first_byte()));
97
98 for value in fan_out.iter() {
99 out.write_all(&value.to_be_bytes())?;
100 }
101
102 progress.inc();
103 let _info = progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN);
104 for entry in &entries_sorted_by_oid {
105 out.write_all(entry.data.id.as_slice())?;
106 }
107
108 progress.inc();
109 let _info = progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN);
110 for entry in &entries_sorted_by_oid {
111 out.write_all(&entry.data.crc32.to_be_bytes())?;
112 }
113
114 progress.inc();
115 let _info = progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN);
116 {
117 let mut offsets64 = Vec::<u64>::new();
118 for entry in &entries_sorted_by_oid {
119 let offset: u32 = if entry.offset > LARGE_OFFSET_THRESHOLD {
120 assert!(
121 offsets64.len() < LARGE_OFFSET_THRESHOLD as usize,
122 "Encoding breakdown - way too many 64bit offsets"
123 );
124 offsets64.push(entry.offset);
125 ((offsets64.len() - 1) as u32) | HIGH_BIT
126 } else {
127 entry.offset as u32
128 };
129 out.write_all(&offset.to_be_bytes())?;
130 }
131 for value in offsets64 {
132 out.write_all(&value.to_be_bytes())?;
133 }
134 }
135
136 out.write_all(pack_hash.as_slice())?;
137
138 let bytes_written_without_trailer = out.bytes;
139 let out = out.inner.into_inner().map_err(io::Error::from)?;
140 let index_hash = out.hash.try_finalize()?;
141 out.inner.write_all(index_hash.as_slice())?;
142 out.inner.flush()?;
143
144 progress.inc();
145 progress.show_throughput_with(
146 start,
147 (bytes_written_without_trailer + object_hash.len_in_bytes() as u64) as usize,
148 progress::bytes().expect("unit always set"),
149 progress::MessageLevel::Success,
150 );
151
152 Ok(index_hash)
153 }
154}
155#[cfg(feature = "streaming-input")]
156pub(crate) use function::write_to;