alice.pds.demo.boltless.dev
This repository has no description
8.2 kB
230 lines
did:plc:3fwecdnvtcscjnrx2p4n7a…
1use std::path::{Path, PathBuf};
2use std::sync::atomic::AtomicBool;
3
4use gix::progress::Discard;
5use knot_git::Repo;
6
7use crate::fsio;
8use crate::{FileCount, MaintError};
9
10const FILE_NAME: &str = "multi-pack-index";
11
12pub(crate) const MIDX_ALLOC_LIMIT_BYTES: usize = 16 * 1024 * 1024;
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum MidxStatus {
16 Written(FileCount),
17 Removed,
18 Absent,
19}
20
21pub(crate) fn clear(objects_dir: &Path) -> Result<(), MaintError> {
22 let pack_dir = objects_dir.join("pack");
23 knot_resource::clear_temps(&pack_dir, FILE_NAME);
24 knot_resource::fsync_path(&pack_dir).map_err(Into::into)
25}
26
27pub fn write(repo: &Repo) -> Result<MidxStatus, MaintError> {
28 let objects_dir = repo.objects_dir();
29 let kind = repo.object_format().kind();
30 let target = objects_dir.join("pack").join(FILE_NAME);
31 let idx_paths = fsio::pack_idx_paths(&objects_dir);
32 match FileCount::new(idx_paths.len()) {
33 count if count.get() >= 2 => {
34 write_atomic(idx_paths, kind, &target)?;
35 Ok(MidxStatus::Written(count))
36 }
37 _ => remove_if_present(&target),
38 }
39}
40
41fn write_atomic(
42 idx_paths: Vec<PathBuf>,
43 kind: gix::hash::Kind,
44 target: &Path,
45) -> Result<(), MaintError> {
46 knot_resource::atomic_write(target, knot_resource::FileMode::Inherited, |file| {
47 let mut writer = std::io::BufWriter::new(file);
48 gix_pack::multi_index::write_from_index_paths(
49 idx_paths,
50 &mut writer,
51 &mut Discard,
52 &AtomicBool::new(false),
53 gix_pack::multi_index::write::Options { object_hash: kind },
54 )
55 .map_err(|e| MaintError::Pack(e.to_string()))?;
56 std::io::Write::flush(&mut writer).map_err(|e| fsio::io_error(target, e))
57 })
58}
59
60fn remove_if_present(target: &Path) -> Result<MidxStatus, MaintError> {
61 match std::fs::remove_file(target) {
62 Ok(()) => {
63 if let Some(dir) = target.parent() {
64 knot_resource::fsync_path(dir)?;
65 }
66 Ok(MidxStatus::Removed)
67 }
68 Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(MidxStatus::Absent),
69 Err(error) => Err(fsio::io_error(target, error)),
70 }
71}
72
73#[cfg(test)]
74mod tests {
75 use knot_git::{Layout, RefUpdate};
76 use knot_types::{BranchName, ObjectFormat, Oid, RefName, RepoDid};
77
78 use super::*;
79 use crate::test_support::{commit_on, empty_tree};
80
81 fn midx_file(repo: &knot_git::Repo) -> PathBuf {
82 repo.objects_dir().join("pack").join(FILE_NAME)
83 }
84
85 fn pack_from(repo: &knot_git::Repo, tip: Oid, kind: gix::hash::Kind) {
86 let closure = repo
87 .select_pack_objects(knot_git::Wants::new(&[tip]), knot_git::Haves::new(&[]))
88 .unwrap();
89 let mut bytes = Vec::new();
90 knot_pack::write_pack(&repo.objects_dir(), closure, None, &mut bytes, kind).unwrap();
91 knot_pack::ingest_pack(
92 &repo.objects_dir(),
93 &bytes,
94 &knot_pack::PackLimits::default(),
95 kind,
96 )
97 .unwrap();
98 }
99
100 fn two_pack_repo(format: ObjectFormat) -> (tempfile::TempDir, knot_git::Repo, Oid, Oid) {
101 let dir = tempfile::tempdir().unwrap();
102 let layout = Layout::new(dir.path())
103 .with_object_format(format)
104 .with_default_branch(BranchName::new("main").unwrap());
105 let did = RepoDid::new("did:plc:limpet").unwrap();
106 let repo = layout.create(&did).unwrap();
107 let kind = format.kind();
108 let first = commit_on(&repo, empty_tree(format), Vec::new(), "scallop");
109 repo.update_ref(&RefUpdate::Create {
110 name: RefName::new("refs/heads/main").unwrap(),
111 new: first,
112 })
113 .unwrap();
114 pack_from(&repo, first, kind);
115 let second = commit_on(&repo, empty_tree(format), Vec::new(), "whelk");
116 repo.update_ref(&RefUpdate::Create {
117 name: RefName::new("refs/heads/side").unwrap(),
118 new: second,
119 })
120 .unwrap();
121 pack_from(&repo, second, kind);
122 (dir, layout.open(&did).unwrap(), first, second)
123 }
124
125 fn assert_two_pack_midx(format: ObjectFormat) {
126 let (_dir, repo, scallop, whelk) = two_pack_repo(format);
127 assert_eq!(fsio::pack_idx_paths(&repo.objects_dir()).len(), 2);
128
129 let status = write(&repo).unwrap();
130 let count = match status {
131 MidxStatus::Written(count) => count,
132 other => panic!("expected a written midx, got {other:?}"),
133 };
134 assert_eq!(count, FileCount::new(2));
135 assert!(midx_file(&repo).exists());
136
137 let parsed =
138 gix_pack::multi_index::File::at(midx_file(&repo), Some(MIDX_ALLOC_LIMIT_BYTES))
139 .unwrap();
140 assert_eq!(parsed.num_indices() as usize, 2);
141 assert!(parsed.num_objects() >= 4);
142
143 assert!(repo.contains(scallop));
144 assert!(repo.contains(whelk));
145 assert_eq!(repo.object_format().kind(), format.kind());
146 }
147
148 #[test]
149 fn writes_a_multi_pack_index_over_two_packs_sha1() {
150 assert_two_pack_midx(ObjectFormat::SHA1);
151 }
152
153 #[test]
154 fn writes_a_multi_pack_index_over_two_packs_sha256() {
155 assert_two_pack_midx(ObjectFormat::SHA256);
156 }
157
158 #[test]
159 fn clear_removes_the_index_and_sidecars_but_keeps_packs() {
160 let dir = tempfile::tempdir().unwrap();
161 let objects_dir = dir.path();
162 let pack_dir = objects_dir.join("pack");
163 std::fs::create_dir_all(&pack_dir).unwrap();
164 let make = |name: &str| std::fs::write(pack_dir.join(name), b"x").unwrap();
165 make(FILE_NAME);
166 make("multi-pack-index-abc.bitmap");
167 make("multi-pack-index-abc.rev");
168 make("pack-scallop.idx");
169 make("pack-scallop.pack");
170
171 clear(objects_dir).unwrap();
172
173 assert!(!pack_dir.join(FILE_NAME).exists());
174 assert!(!pack_dir.join("multi-pack-index-abc.bitmap").exists());
175 assert!(!pack_dir.join("multi-pack-index-abc.rev").exists());
176 assert!(
177 pack_dir.join("pack-scallop.idx").exists(),
178 "real packs are left in place"
179 );
180 assert!(pack_dir.join("pack-scallop.pack").exists());
181 }
182
183 #[test]
184 fn lookup_resolves_through_the_midx_once_idx_files_are_gone() {
185 let (_dir, repo, scallop, whelk) = two_pack_repo(ObjectFormat::SHA1);
186 let objects_dir = repo.objects_dir();
187 assert!(matches!(write(&repo).unwrap(), MidxStatus::Written(_)));
188
189 fsio::loose_objects(&objects_dir)
190 .iter()
191 .for_each(|(_, path)| std::fs::remove_file(path).unwrap());
192 fsio::pack_idx_paths(&objects_dir)
193 .iter()
194 .for_each(|idx| std::fs::remove_file(idx).unwrap());
195 assert!(
196 midx_file(&repo).exists(),
197 "the multi-pack-index is the only index left on disk"
198 );
199
200 let via_midx = knot_git::Repo::open(repo.git().git_dir()).unwrap();
201 assert!(
202 via_midx.contains(scallop) && via_midx.contains(whelk),
203 "objects resolve through the multi-pack-index with no per-pack idx present"
204 );
205
206 std::fs::remove_file(midx_file(&repo)).unwrap();
207 let bare = knot_git::Repo::open(repo.git().git_dir()).unwrap();
208 assert!(
209 !bare.contains(scallop) && !bare.contains(whelk),
210 "with the index gone the packs are unreadable, proving the midx served the lookup"
211 );
212 }
213
214 #[test]
215 fn fewer_than_two_packs_writes_nothing_and_clears_stale() {
216 let dir = tempfile::tempdir().unwrap();
217 let layout = Layout::new(dir.path()).with_default_branch(BranchName::new("main").unwrap());
218 let did = RepoDid::new("did:plc:conch").unwrap();
219 let repo = layout.create(&did).unwrap();
220
221 assert_eq!(write(&repo).unwrap(), MidxStatus::Absent);
222 assert!(!midx_file(&repo).exists());
223
224 let target = midx_file(&repo);
225 std::fs::create_dir_all(target.parent().unwrap()).unwrap();
226 std::fs::write(&target, b"stale").unwrap();
227 assert_eq!(write(&repo).unwrap(), MidxStatus::Removed);
228 assert!(!midx_file(&repo).exists());
229 }
230}