alice.pds.demo.boltless.dev
This repository has no description
1use std::cmp::Ordering;
2
3pub(crate) const LARGE_OFFSET_THRESHOLD: u64 = 0x7fff_ffff;
4pub(crate) const HIGH_BIT: u32 = 0x8000_0000;
5
6pub(crate) fn fanout(iter: &mut dyn ExactSizeIterator<Item = u8>) -> [u32; 256] {
7 let mut fan_out = [0u32; 256];
8 let entries_len = iter.len() as u32;
9 let mut iter = iter.enumerate();
10 let mut idx_and_entry = iter.next();
11 let mut upper_bound = 0;
12
13 for (offset_be, byte) in fan_out.iter_mut().zip(0u8..=255) {
14 *offset_be = match idx_and_entry.as_ref() {
15 Some((_idx, first_byte)) => match first_byte.cmp(&byte) {
16 Ordering::Less => {
17 unreachable!("ids should be ordered, and we make sure to keep ahead with them")
18 }
19 Ordering::Greater => upper_bound,
20 Ordering::Equal => {
21 if byte == 255 {
22 entries_len
23 } else {
24 idx_and_entry = iter.find(|(_, first_byte)| *first_byte != byte);
25 upper_bound = idx_and_entry
26 .as_ref()
27 .map_or(entries_len, |(idx, _)| *idx as u32);
28 upper_bound
29 }
30 }
31 },
32 None => entries_len,
33 };
34 }
35
36 fan_out
37}
38
39#[cfg(feature = "streaming-input")]
40mod function {
41 use std::io;
42
43 use gix_features::progress::{self, DynNestedProgress};
44
45 use super::{HIGH_BIT, LARGE_OFFSET_THRESHOLD, fanout};
46 use crate::index::V2_SIGNATURE;
47
48 struct Count<W> {
49 bytes: u64,
50 inner: W,
51 }
52
53 impl<W> Count<W> {
54 fn new(inner: W) -> Self {
55 Count { bytes: 0, inner }
56 }
57 }
58
59 impl<W> io::Write for Count<W>
60 where
61 W: io::Write,
62 {
63 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
64 let written = self.inner.write(buf)?;
65 self.bytes += written as u64;
66 Ok(written)
67 }
68
69 fn flush(&mut self) -> io::Result<()> {
70 self.inner.flush()
71 }
72 }
73
74 pub(crate) fn write_to(
75 out: &mut dyn io::Write,
76 entries_sorted_by_oid: Vec<crate::cache::delta::Item<crate::index::write::TreeEntry>>,
77 pack_hash: &gix_hash::ObjectId,
78 kind: crate::index::Version,
79 object_hash: gix_hash::Kind,
80 progress: &mut dyn DynNestedProgress,
81 ) -> Result<gix_hash::ObjectId, gix_hash::io::Error> {
82 use io::Write;
83 assert_eq!(
84 kind,
85 crate::index::Version::V2,
86 "Can only write V2 packs right now"
87 );
88 assert!(
89 entries_sorted_by_oid.len() <= u32::MAX as usize,
90 "a pack cannot have more than u32::MAX objects"
91 );
92
93 // Write header
94 let mut out = Count::new(std::io::BufWriter::with_capacity(
95 8 * 4096,
96 gix_hash::io::Write::new(out, object_hash),
97 ));
98 out.write_all(V2_SIGNATURE)?;
99 out.write_all(&(kind as u32).to_be_bytes())?;
100
101 progress.init(Some(4), progress::steps());
102 let start = std::time::Instant::now();
103 let _info = progress.add_child_with_id(
104 "writing fan-out table".into(),
105 gix_features::progress::UNKNOWN,
106 );
107 let fan_out = fanout(&mut entries_sorted_by_oid.iter().map(|e| e.data.id.first_byte()));
108
109 for value in fan_out.iter() {
110 out.write_all(&value.to_be_bytes())?;
111 }
112
113 progress.inc();
114 let _info =
115 progress.add_child_with_id("writing ids".into(), gix_features::progress::UNKNOWN);
116 for entry in &entries_sorted_by_oid {
117 out.write_all(entry.data.id.as_slice())?;
118 }
119
120 progress.inc();
121 let _info =
122 progress.add_child_with_id("writing crc32".into(), gix_features::progress::UNKNOWN);
123 for entry in &entries_sorted_by_oid {
124 out.write_all(&entry.data.crc32.to_be_bytes())?;
125 }
126
127 progress.inc();
128 let _info =
129 progress.add_child_with_id("writing offsets".into(), gix_features::progress::UNKNOWN);
130 {
131 let mut offsets64 = Vec::<u64>::new();
132 for entry in &entries_sorted_by_oid {
133 let offset: u32 = if entry.offset > LARGE_OFFSET_THRESHOLD {
134 assert!(
135 offsets64.len() < LARGE_OFFSET_THRESHOLD as usize,
136 "Encoding breakdown - way too many 64bit offsets"
137 );
138 offsets64.push(entry.offset);
139 ((offsets64.len() - 1) as u32) | HIGH_BIT
140 } else {
141 entry.offset as u32
142 };
143 out.write_all(&offset.to_be_bytes())?;
144 }
145 for value in offsets64 {
146 out.write_all(&value.to_be_bytes())?;
147 }
148 }
149
150 out.write_all(pack_hash.as_slice())?;
151
152 let bytes_written_without_trailer = out.bytes;
153 let out = out.inner.into_inner().map_err(io::Error::from)?;
154 let index_hash = out.hash.try_finalize()?;
155 out.inner.write_all(index_hash.as_slice())?;
156 out.inner.flush()?;
157
158 progress.inc();
159 progress.show_throughput_with(
160 start,
161 (bytes_written_without_trailer + object_hash.len_in_bytes() as u64) as usize,
162 progress::bytes().expect("unit always set"),
163 progress::MessageLevel::Success,
164 );
165
166 Ok(index_hash)
167 }
168}
169#[cfg(feature = "streaming-input")]
170pub(crate) use function::write_to;