rad:z3gqcJUoA1n9HaHKufZs5FCSGazv5 heartwoodfb9b39f428afe6428f24ec0db0ad4094b1ca9764
{
"request": "trigger",
"version": 1,
"event_type": "patch",
"repository": {
"id": "rad:z3gqcJUoA1n9HaHKufZs5FCSGazv5",
"name": "heartwood",
"description": "Radicle Heartwood Protocol & Stack",
"private": false,
"default_branch": "master",
"delegates": [
"did:key:z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT",
"did:key:z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM",
"did:key:z6MkkPvBfjP4bQmco5Dm7UGsX2ruDBieEHi8n9DVJWX5sTEz"
]
},
"action": "Created",
"patch": {
"id": "4f2e158e100c17346b33fa366014e55cc503336b",
"author": {
"id": "did:key:z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM",
"alias": null
},
"title": "fetch: remove .keep files",
"state": {
"status": "merged",
"conflicts": []
},
"before": "cb2cbf014a123a1cd0cd0f188f093e014f93647b",
"after": "fb9b39f428afe6428f24ec0db0ad4094b1ca9764",
"commits": [
"fb9b39f428afe6428f24ec0db0ad4094b1ca9764"
],
"target": "3fc92f83fe089263c470b328414d923227cf0e4a",
"labels": [],
"assignees": [],
"revisions": [
{
"id": "4f2e158e100c17346b33fa366014e55cc503336b",
"author": {
"id": "did:key:z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM",
"alias": null
},
"description": "The `.keep` files for packs were never being removed, resulting in all pack\nfiles never being cleaned up by `git gc`.\n\nReferring to Gitoxide[[0]][[1]], the same idea for removing the `.keep` file is\napplied when performing `write_pack`.\n\n[0]: https://github.com/Byron/gitoxide/blob/c4e0a82ec9948ed0d9876a50b156d1ba30e2ad77/gix/src/remote/connection/fetch/receive_pack.rs#L335-L341\n[1]: https://github.com/Byron/gitoxide/blob/c4e0a82ec9948ed0d9876a50b156d1ba30e2ad77/gix-pack/src/bundle/write/types.rs#L47-L51\n\nSigned-off-by: Fintan Halpenny <fintan.halpenny@gmail.com>\nX-Clacks-Overhead: GNU Terry Pratchett",
"base": "25c6660a59c621de1ec406e4f9b2ccfbd21e1b90",
"oid": "27377c53fcf272147585d22fcbdf107abbf4731c",
"timestamp": 1715689935
},
{
"id": "09ca1a4976c368a30910f08760e08b7125c1c6f2",
"author": {
"id": "did:key:z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM",
"alias": null
},
"description": "Changes:\n- Use `Keepfile` type to implement `Drop` semantics for removing `.keep` files\n- Adds a check to an existing test to ensure that semantics are correct",
"base": "cb2cbf014a123a1cd0cd0f188f093e014f93647b",
"oid": "7ecddebbcb115fe9d86b8da60fbd633b1ad585d6",
"timestamp": 1715856400
},
{
"id": "08a29572106939392526798bda17e31cb21ea0d3",
"author": {
"id": "did:key:z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM",
"alias": null
},
"description": "Changes:\n- simplify extension check\n- fix documentation typo",
"base": "cb2cbf014a123a1cd0cd0f188f093e014f93647b",
"oid": "fb9b39f428afe6428f24ec0db0ad4094b1ca9764",
"timestamp": 1715866406
},
{
"id": "808b3bae4d527fe570289a9ac9d6c80ef3d7e6ef",
"author": {
"id": "did:key:z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM",
"alias": null
},
"description": "Rebase.",
"base": "ca7db1620c6f8d4e9caed0902b191c5e6def8bc3",
"oid": "d56d619f5601ba456c7bb517be96a1418743cf9e",
"timestamp": 1715867359
}
]
}
}
{
"response": "triggered",
"run_id": {
"id": "8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e"
},
"info_url": "https://cci.rad.levitte.org//8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e.html"
}
Started at: 2026-09-07 21:26:01.616608+02:00
Commands:
$ rad clone rad:z3gqcJUoA1n9HaHKufZs5FCSGazv5 .
✓ Creating checkout in ./...
✓ Remote cloudhead@z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT added
✓ Remote-tracking branch cloudhead@z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT/master created for z6MksFqXN3Yhqk8pTJdUGLwATkRfQvwZXPqR2qMEhbS9wzpT
✓ Remote fintohaps@z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM added
✓ Remote-tracking branch fintohaps@z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM/master created for z6MkireRatUThvd3qzfKht1S44wpm4FEWSSa4PRMTSQZ3voM
✓ Remote lorenz@z6MkkPvBfjP4bQmco5Dm7UGsX2ruDBieEHi8n9DVJWX5sTEz added
✓ Remote-tracking branch lorenz@z6MkkPvBfjP4bQmco5Dm7UGsX2ruDBieEHi8n9DVJWX5sTEz/master created for z6MkkPvBfjP4bQmco5Dm7UGsX2ruDBieEHi8n9DVJWX5sTEz
✓ Repository successfully cloned under /opt/radcis/ci.rad.levitte.org/cci/state/8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e/w/
╭────────────────────────────────────╮
│ heartwood │
│ Radicle Heartwood Protocol & Stack │
│ 189 issues · 38 patches │
╰────────────────────────────────────╯
Run `cd ./.` to go to the repository directory.
Exit code: 0
$ rad patch checkout 4f2e158e100c17346b33fa366014e55cc503336b
✓ Switched to branch patch/4f2e158 at revision 08a2957
✓ Branch patch/4f2e158 setup to track rad/patches/4f2e158e100c17346b33fa366014e55cc503336b
Exit code: 0
$ git config advice.detachedHead false
Exit code: 0
$ git checkout fb9b39f428afe6428f24ec0db0ad4094b1ca9764
HEAD is now at fb9b39f4 fetch: remove `.keep` files
Exit code: 0
$ rad patch show 4f2e158e100c17346b33fa366014e55cc503336b -p
╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮
│ Title fetch: remove .keep files │
│ Patch 4f2e158e100c17346b33fa366014e55cc503336b │
│ Author fintohaps z6Mkire…SQZ3voM │
│ Head fb9b39f428afe6428f24ec0db0ad4094b1ca9764 │
│ Base cb2cbf014a123a1cd0cd0f188f093e014f93647b │
│ Target master │
│ Branches patch/4f2e158 │
│ Commits ahead 1, behind 1345 │
│ Status merged │
│ │
│ The `.keep` files for packs were never being removed, resulting in all pack │
│ files never being cleaned up by `git gc`. │
│ │
│ Referring to Gitoxide[[0]][[1]], the same idea for removing the `.keep` file is │
│ applied when performing `write_pack`. │
│ │
│ [0]: │
│ https://github.com/Byron/gitoxide/blob/c4e0a82ec9948ed0d9876a50b156d1ba30e2ad77/gix/src/remote/connection/fetch/receive_pack.rs#L335-L341 │
│ [1]: │
│ https://github.com/Byron/gitoxide/blob/c4e0a82ec9948ed0d9876a50b156d1ba30e2ad77/gix-pack/src/bundle/write/types.rs#L47-L51 │
│ │
│ Signed-off-by: Fintan Halpenny <fintan.halpenny@gmail.com> │
│ X-Clacks-Overhead: GNU Terry Pratchett │
├───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ fb9b39f fetch: remove `.keep` files │
├───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ● Revision 4f2e158 @ 25c6660..27377c5 by fintohaps z6Mkire…SQZ3voM 2 years ago │
│ ↑ Revision 09ca1a4 @ cb2cbf0..7ecddeb by fintohaps z6Mkire…SQZ3voM 2 years ago │
│ ↑ Revision 08a2957 @ cb2cbf0..fb9b39f by fintohaps z6Mkire…SQZ3voM 2 years ago │
│ ↑ Revision 808b3ba @ ca7db16..d56d619 by cloudhead z6MksFq…bS9wzpT 2 years ago │
│ └─ ✓ merged by cloudhead z6MksFq…bS9wzpT │
╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
commit fb9b39f428afe6428f24ec0db0ad4094b1ca9764
Author: Fintan Halpenny <fintan.halpenny@gmail.com>
Date: Thu May 16 11:30:34 2024 +0100
fetch: remove `.keep` files
A `.keep` file in the `objects/pack` directory acts as a lock to ensure that
operations like `git gc` do not clean up `.idx` files while a fetch is being
performed. It is safe to remove the `.keep` file once a fetch has completed and
a reference points to a given object in the packfile.
To ensure this happens in `radicle-fetch`, a newtype `Keepfile` is introduced.
It uses special `Drop` semantics by attempting to remove the file when the
object is dropped -- logging a warning if it fails to do so.
`FetchState` is augmented to keep a set of the `Keepfile`s. This ensures that
when the fetch completes, either successfully or with a failure, when
`FetchState` is dropped then so are the `Keepfile`s.
We ensure that the expected behaviour of removing the `.keep`files occurs by
adding a check in `e2e::test_replication` to check Alice's repository's packs.
Signed-off-by: Fintan Halpenny <fintan.halpenny@gmail.com>
X-Clacks-Overhead: GNU Terry Pratchett
diff --git a/radicle-fetch/src/git.rs b/radicle-fetch/src/git.rs
index eafe93305..b7b4c42ff 100644
--- a/radicle-fetch/src/git.rs
+++ b/radicle-fetch/src/git.rs
@@ -1,5 +1,6 @@
pub(crate) mod mem;
pub(crate) mod repository;
+pub(crate) mod packfile;
pub mod refs;
diff --git a/radicle-fetch/src/git/packfile.rs b/radicle-fetch/src/git/packfile.rs
new file mode 100644
index 000000000..1b7a6612b
--- /dev/null
+++ b/radicle-fetch/src/git/packfile.rs
@@ -0,0 +1,32 @@
+use std::fs;
+use std::path::{Path, PathBuf};
+
+/// The [`PathBuf`] which points to a `*.keep` file, which should correspond to
+/// a packfile.
+///
+/// Upon drop, it attempts to remove the [`PathBuf`] to release the lock on the
+/// packfile index, allowing it to be garbage collected.
+#[derive(Clone, Debug)]
+pub struct Keepfile {
+ path: PathBuf,
+}
+
+impl Keepfile {
+ pub fn new<P: AsRef<Path>>(path: P) -> Option<Self> {
+ let path = path.as_ref();
+ match path.extension() {
+ Some(ext) if ext == "keep" => Some(Self {
+ path: path.to_path_buf(),
+ }),
+ _ => None,
+ }
+ }
+}
+
+impl Drop for Keepfile {
+ fn drop(&mut self) {
+ if let Err(e) = fs::remove_file(&self.path) {
+ log::warn!(target: "fetch", "Failed to remove {:?}: {e}", self.path);
+ }
+ }
+}
diff --git a/radicle-fetch/src/state.rs b/radicle-fetch/src/state.rs
index 5c944ecd9..489cbb832 100644
--- a/radicle-fetch/src/state.rs
+++ b/radicle-fetch/src/state.rs
@@ -14,6 +14,7 @@ use radicle::storage::{
};
use crate::git;
+use crate::git::packfile::Keepfile;
use crate::git::refs::{Applied, Update};
use crate::git::repository;
use crate::sigrefs::SignedRefsAt;
@@ -160,6 +161,10 @@ pub struct FetchState {
sigrefs: SigrefTips,
/// Seen reference tips, per remote.
tips: BTreeMap<PublicKey, Vec<Update<'static>>>,
+ /// The `.keep` files created during packfile transfers. They are kept
+ /// within the state, so that when the state is dropped, it also attempts to
+ /// delete the files to release the locks on the packfiles.
+ keepfiles: Vec<Keepfile>,
}
impl FetchState {
@@ -233,9 +238,11 @@ impl FetchState {
let wants_haves = step.wants_haves(&handle.repo, &refs)?;
if !wants_haves.wants.is_empty() {
- handle
- .transport
- .fetch(wants_haves, handle.interrupt.clone(), handshake)?;
+ let keepfile =
+ handle
+ .transport
+ .fetch(wants_haves, handle.interrupt.clone(), handshake)?;
+ self.keepfiles.extend(keepfile);
} else {
log::trace!(target: "fetch", "Nothing to fetch")
};
diff --git a/radicle-fetch/src/transport.rs b/radicle-fetch/src/transport.rs
index 98c411b73..a577452f5 100644
--- a/radicle-fetch/src/transport.rs
+++ b/radicle-fetch/src/transport.rs
@@ -21,6 +21,7 @@ use radicle::storage::git::Repository;
use thiserror::Error;
use crate::git::oid;
+use crate::git::packfile::Keepfile;
use crate::git::repository;
/// Open a reader and writer stream to pass to the ls-refs and fetch
@@ -120,7 +121,7 @@ where
wants_haves: WantsHaves,
interrupt: Arc<AtomicBool>,
handshake: &handshake::Outcome,
- ) -> io::Result<()> {
+ ) -> io::Result<Option<Keepfile>> {
log::trace!(
target: "fetch",
"Running fetch wants={:?}, haves={:?}",
@@ -170,7 +171,7 @@ where
}
}
- Ok(())
+ Ok(out.keepfile)
}
/// Signal to the server side that we are done sending ls-refs and
diff --git a/radicle-fetch/src/transport/fetch.rs b/radicle-fetch/src/transport/fetch.rs
index f56d1dadb..cc55450df 100644
--- a/radicle-fetch/src/transport/fetch.rs
+++ b/radicle-fetch/src/transport/fetch.rs
@@ -17,6 +17,8 @@ use gix_transport::client;
use gix_transport::client::{ExtendedBufRead, MessageKind};
use gix_transport::Protocol;
+use crate::git::packfile;
+
use super::{agent_name, indicate_end_of_interaction, Connection, WantsHaves};
pub type Error = gix_protocol::fetch::Error;
@@ -102,6 +104,7 @@ pub struct Fetch {
pub struct FetchOut {
pub refs: Vec<Ref>,
pub pack: Option<pack::bundle::write::Outcome>,
+ pub keepfile: Option<packfile::Keepfile>,
}
// FIXME: the delegate pattern will be removed in the near future and
@@ -127,6 +130,7 @@ impl<'a> Delegate for &'a mut Fetch {
.pack_writer
.write_pack(input, progress)
.map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
+ self.out.keepfile = pack.keep_path.as_ref().and_then(packfile::Keepfile::new);
self.out.pack = Some(pack);
Ok(())
}
@@ -204,6 +208,7 @@ where
out: FetchOut {
refs: Vec::new(),
pack: None,
+ keepfile: None,
},
};
diff --git a/radicle-node/src/tests/e2e.rs b/radicle-node/src/tests/e2e.rs
index ccdc9662b..af77677b7 100644
--- a/radicle-node/src/tests/e2e.rs
+++ b/radicle-node/src/tests/e2e.rs
@@ -216,6 +216,21 @@ fn test_replication() {
alice.storage.repository(acme).unwrap().validate(),
Ok(validations) if validations.is_empty()
);
+
+ // Ensure that .keep files are deleted upon replication
+ {
+ let repo = alice.storage.repository(acme).unwrap();
+ let pack_dir = repo.path().join("objects").join("pack");
+ for entry in std::fs::read_dir(pack_dir).unwrap() {
+ let entry = entry.unwrap();
+ let path = entry.path();
+ assert_ne!(
+ path.extension(),
+ Some("keep".as_ref()),
+ "found .keep file after fetch: {path:?}"
+ );
+ }
+ }
}
#[test]
Exit code: 0
shell: 'cargo --version rustc --version cargo fmt --check cargo clippy --all-targets --workspace -- --deny clippy::all cargo build --all-targets --workspace cargo doc --workspace cargo test --workspace --no-fail-fast '
Commands:
$ podman run --name 8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e -v /opt/radcis/ci.rad.levitte.org/cci/state/8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e/s:/8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e/s:ro -v /opt/radcis/ci.rad.levitte.org/cci/state/8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e/w:/8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e/w -w /8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e/w -v /opt/radcis/ci.rad.levitte.org/.radicle:/${id}/.radicle:ro -e RAD_HOME=/${id}/.radicle rust:trixie bash /8c67ef57-d10e-4d2c-bb69-2c0b08bf1c3e/s/script.sh
+ cargo --version
info: syncing channel updates for '1.77-x86_64-unknown-linux-gnu'
info: latest update on 2024-04-09, rust version 1.77.2 (25ef9e3d8 2024-04-09)
info: downloading component 'cargo'
info: downloading component 'rust-std'
info: downloading component 'rustc'
info: installing component 'cargo'
info: installing component 'rust-std'
info: installing component 'rustc'
cargo 1.77.2 (e52e36006 2024-03-26)
+ rustc --version
rustc 1.77.2 (25ef9e3d8 2024-04-09)
+ cargo fmt --check
error: 'cargo-fmt' is not installed for the toolchain '1.77-x86_64-unknown-linux-gnu'.
To install, run `rustup component add --toolchain 1.77-x86_64-unknown-linux-gnu rustfmt`
Exit code: 1
{
"response": "finished",
"result": "failure"
}