Seperated build directory from target directory

This commits implements the seperation of the intermidate artifact directory (called "build directory") from the target directory. (see #14125)

Author: ranger-ross

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -217,7 +217,7 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
  } else if unit.target.is_custom_build() {
  self.build_script_dir(unit)
  } else if unit.target.is_example() {
- self.layout(unit.kind).examples().to_path_buf()
+ self.layout(unit.kind).build_examples().to_path_buf()
  } else if unit.artifact.is_true() {
  self.artifact_dir(unit)
  } else {

src/cargo/core/compiler/custom_build.rs

@@ -1124,7 +1124,7 @@ fn prepare_metabuild(
  let path = unit
  .pkg
  .manifest()
- .metabuild_path(build_runner.bcx.ws.target_dir());
+ .metabuild_path(build_runner.bcx.ws.build_dir());
  paths::create_dir_all(path.parent().unwrap())?;
  paths::write_if_changed(path, &output)?;
  Ok(())

src/cargo/core/compiler/fingerprint/dep_info.rs

@@ -51,9 +51,9 @@ pub struct RustcDepInfo {
 pub enum DepInfoPathType {
  /// src/, e.g. src/lib.rs
  PackageRootRelative,
- /// target/debug/deps/lib...
+ /// {build-dir}/debug/deps/lib...
  /// or an absolute path /.../sysroot/...
- TargetRootRelative,
+ BuildRootRelative,
 }
 
 /// Same as [`RustcDepInfo`] except avoids absolute paths as much as possible to
@@ -126,7 +126,7 @@ impl EncodedDepInfo {
  for _ in 0..nfiles {
  let ty = match read_u8(bytes)? {
  0 => DepInfoPathType::PackageRootRelative,
- 1 => DepInfoPathType::TargetRootRelative,
+ 1 => DepInfoPathType::BuildRootRelative,
  _ => return None,
  };
  let path_bytes = read_bytes(bytes)?;
@@ -210,7 +210,7 @@ impl EncodedDepInfo {
  for (ty, file, checksum_info) in self.files.iter() {
  match ty {
  DepInfoPathType::PackageRootRelative => dst.push(0),
- DepInfoPathType::TargetRootRelative => dst.push(1),
+ DepInfoPathType::BuildRootRelative => dst.push(1),
  }
  write_bytes(dst, paths::path2bytes(file)?);
  write_bool(dst, checksum_info.is_some());
@@ -292,14 +292,14 @@ pub fn translate_dep_info(
  cargo_dep_info: &Path,
  rustc_cwd: &Path,
  pkg_root: &Path,
- target_root: &Path,
+ build_root: &Path,
  rustc_cmd: &ProcessBuilder,
  allow_package: bool,
  env_config: &Arc<HashMap<String, OsString>>,
 ) -> CargoResult<()> {
  let depinfo = parse_rustc_dep_info(rustc_dep_info)?;
 
- let target_root = crate::util::try_canonicalize(target_root)?;
+ let build_root = crate::util::try_canonicalize(build_root)?;
  let pkg_root = crate::util::try_canonicalize(pkg_root)?;
  let mut on_disk_info = EncodedDepInfo::default();
  on_disk_info.env = depinfo.env;
@@ -351,8 +351,8 @@ pub fn translate_dep_info(
  let canon_file =
  crate::util::try_canonicalize(&abs_file).unwrap_or_else(|_| abs_file.clone());
 
- let (ty, path) = if let Ok(stripped) = canon_file.strip_prefix(&target_root) {
- (DepInfoPathType::TargetRootRelative, stripped)
+ let (ty, path) = if let Ok(stripped) = canon_file.strip_prefix(&build_root) {
+ (DepInfoPathType::BuildRootRelative, stripped)
  } else if let Ok(stripped) = canon_file.strip_prefix(&pkg_root) {
  if !allow_package {
  return None;
@@ -362,7 +362,7 @@ pub fn translate_dep_info(
  // It's definitely not target root relative, but this is an absolute path (since it was
  // joined to rustc_cwd) and as such re-joining it later to the target root will have no
  // effect.
- (DepInfoPathType::TargetRootRelative, &*abs_file)
+ (DepInfoPathType::BuildRootRelative, &*abs_file)
  };
  Some((ty, path.to_owned()))
  };
@@ -472,7 +472,7 @@ pub fn parse_rustc_dep_info(rustc_dep_info: &Path) -> CargoResult<RustcDepInfo>
 /// indicates that the crate should likely be rebuilt.
 pub fn parse_dep_info(
  pkg_root: &Path,
- target_root: &Path,
+ build_root: &Path,
  dep_info: &Path,
 ) -> CargoResult<Option<RustcDepInfo>> {
  let Ok(data) = paths::read_bytes(dep_info) else {
@@ -487,7 +487,7 @@ pub fn parse_dep_info(
  ret.files
  .extend(info.files.into_iter().map(|(ty, path, checksum_info)| {
  (
- make_absolute_path(ty, pkg_root, target_root, path),
+ make_absolute_path(ty, pkg_root, build_root, path),
  checksum_info.and_then(|(file_len, checksum)| {
  Checksum::from_str(&checksum).ok().map(|c| (file_len, c))
  }),
@@ -499,13 +499,13 @@ pub fn parse_dep_info(
 fn make_absolute_path(
  ty: DepInfoPathType,
  pkg_root: &Path,
- target_root: &Path,
+ build_root: &Path,
  path: PathBuf,
 ) -> PathBuf {
  match ty {
  DepInfoPathType::PackageRootRelative => pkg_root.join(path),
  // N.B. path might be absolute here in which case the join will have no effect
- DepInfoPathType::TargetRootRelative => target_root.join(path),
+ DepInfoPathType::BuildRootRelative => build_root.join(path),
  }
 }
 
@@ -678,7 +678,7 @@ mod encoded_dep_info {
  fn gen_test(checksum: bool) {
  let checksum = checksum.then_some((768, "c01efc669f09508b55eced32d3c88702578a7c3e".into()));
  let lib_rs = (
- DepInfoPathType::TargetRootRelative,
+ DepInfoPathType::BuildRootRelative,
  PathBuf::from("src/lib.rs"),
  checksum.clone(),
  );
@@ -691,7 +691,7 @@ mod encoded_dep_info {
  assert_eq!(EncodedDepInfo::parse(&data).unwrap(), depinfo);
 
  let mod_rs = (
- DepInfoPathType::TargetRootRelative,
+ DepInfoPathType::BuildRootRelative,
  PathBuf::from("src/mod.rs"),
  checksum.clone(),
  );

src/cargo/core/compiler/fingerprint/mod.rs

@@ -839,7 +839,7 @@ impl LocalFingerprint {
  mtime_cache: &mut HashMap<PathBuf, FileTime>,
  checksum_cache: &mut HashMap<PathBuf, Checksum>,
  pkg: &Package,
- target_root: &Path,
+ build_root: &Path,
  cargo_exe: &Path,
  gctx: &GlobalContext,
  ) -> CargoResult<Option<StaleItem>> {
@@ -852,8 +852,8 @@ impl LocalFingerprint {
  // the `dep_info` file itself whose mtime represents the start of
  // rustc.
  LocalFingerprint::CheckDepInfo { dep_info, checksum } => {
- let dep_info = target_root.join(dep_info);
- let Some(info) = parse_dep_info(pkg_root, target_root, &dep_info)? else {
+ let dep_info = build_root.join(dep_info);
+ let Some(info) = parse_dep_info(pkg_root, build_root, &dep_info)? else {
  return Ok(Some(StaleItem::MissingFile(dep_info)));
  };
  for (key, previous) in info.env.iter() {
@@ -910,7 +910,7 @@ impl LocalFingerprint {
  LocalFingerprint::RerunIfChanged { output, paths } => Ok(find_stale_file(
  mtime_cache,
  checksum_cache,
- &target_root.join(output),
+ &build_root.join(output),
  paths.iter().map(|p| (pkg_root.join(p), None)),
  false,
  )),
@@ -1153,7 +1153,7 @@ impl Fingerprint {
  mtime_cache: &mut HashMap<PathBuf, FileTime>,
  checksum_cache: &mut HashMap<PathBuf, Checksum>,
  pkg: &Package,
- target_root: &Path,
+ build_root: &Path,
  cargo_exe: &Path,
  gctx: &GlobalContext,
  ) -> CargoResult<()> {
@@ -1261,7 +1261,7 @@ impl Fingerprint {
  mtime_cache,
  checksum_cache,
  pkg,
- target_root,
+ build_root,
  cargo_exe,
  gctx,
  )? {
@@ -1449,13 +1449,13 @@ fn calculate(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult
 
  // After we built the initial `Fingerprint` be sure to update the
  // `fs_status` field of it.
- let target_root = target_root(build_runner);
+ let build_root = build_root(build_runner);
  let cargo_exe = build_runner.bcx.gctx.cargo_exe()?;
  fingerprint.check_filesystem(
  &mut build_runner.mtime_cache,
  &mut build_runner.checksum_cache,
  &unit.pkg,
- &target_root,
+ &build_root,
  cargo_exe,
  build_runner.bcx.gctx,
  )?;
@@ -1493,7 +1493,7 @@ fn calculate_normal(
  };
 
  // Afterwards calculate our own fingerprint information.
- let target_root = target_root(build_runner);
+ let build_root = build_root(build_runner);
  let local = if unit.mode.is_doc() || unit.mode.is_doc_scrape() {
  // rustdoc does not have dep-info files.
  let fingerprint = pkg_fingerprint(build_runner.bcx, &unit.pkg).with_context(|| {
@@ -1505,7 +1505,7 @@ fn calculate_normal(
  vec![LocalFingerprint::Precalculated(fingerprint)]
  } else {
  let dep_info = dep_info_loc(build_runner, unit);
- let dep_info = dep_info.strip_prefix(&target_root).unwrap().to_path_buf();
+ let dep_info = dep_info.strip_prefix(&build_root).unwrap().to_path_buf();
  vec![LocalFingerprint::CheckDepInfo {
  dep_info,
  checksum: build_runner.bcx.gctx.cli_unstable().checksum_freshness,
@@ -1714,7 +1714,7 @@ fn build_script_local_fingerprints(
  // longstanding bug, in Cargo. Recent refactorings just made it painfully
  // obvious.
  let pkg_root = unit.pkg.root().to_path_buf();
- let target_dir = target_root(build_runner);
+ let build_dir = build_root(build_runner);
  let env_config = Arc::clone(build_runner.bcx.gctx.env_config()?);
  let calculate =
  move |deps: &BuildDeps, pkg_fingerprint: Option<&dyn Fn() -> CargoResult<String>>| {
@@ -1747,7 +1747,7 @@ fn build_script_local_fingerprints(
  // them all here.
  Ok(Some(local_fingerprints_deps(
  deps,
- &target_dir,
+ &build_dir,
  &pkg_root,
  &env_config,
  )))
@@ -1783,7 +1783,7 @@ fn build_script_override_fingerprint(
 /// [`RunCustomBuild`]: crate::core::compiler::CompileMode::RunCustomBuild
 fn local_fingerprints_deps(
  deps: &BuildDeps,
- target_root: &Path,
+ build_root: &Path,
  pkg_root: &Path,
  env_config: &Arc<HashMap<String, OsString>>,
 ) -> Vec<LocalFingerprint> {
@@ -1796,7 +1796,7 @@ fn local_fingerprints_deps(
  // absolute prefixes from them.
  let output = deps
  .build_script_output
- .strip_prefix(target_root)
+ .strip_prefix(build_root)
  .unwrap()
  .to_path_buf();
  let paths = deps
@@ -1854,10 +1854,10 @@ pub fn dep_info_loc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> Path
  build_runner.files().fingerprint_file_path(unit, "dep-")
 }
 
-/// Returns an absolute path that target directory.
+/// Returns an absolute path that build directory.
 /// All paths are rewritten to be relative to this.
-fn target_root(build_runner: &BuildRunner<'_, '_>) -> PathBuf {
- build_runner.bcx.ws.target_dir().into_path_unlocked()
+fn build_root(build_runner: &BuildRunner<'_, '_>) -> PathBuf {
+ build_runner.bcx.ws.build_dir().into_path_unlocked()
 }
 
 /// Reads the value from the old fingerprint hash file and compare.

src/cargo/core/compiler/layout.rs

@@ -128,13 +128,20 @@ pub struct Layout {
  fingerprint: PathBuf,
  /// The directory for examples: `$dest/examples`
  examples: PathBuf,
+ /// The directory for pre-uplifted examples: `$build-dir/debug/examples`
+ build_examples: PathBuf,
  /// The directory for rustdoc output: `$root/doc`
  doc: PathBuf,
  /// The directory for temporary data of integration tests and benches: `$dest/tmp`
  tmp: PathBuf,
  /// The lockfile for a build (`.cargo-lock`). Will be unlocked when this
  /// struct is `drop`ped.
  _lock: FileLock,
+ /// Same as `_lock` but for the build directory.
+ ///
+ /// Will be `None` when the build-dir and target-dir are the same path as we cannot
+ /// lock the same path twice.
+ _build_lock: Option<FileLock>,
 }
 
 impl Layout {
@@ -150,15 +157,22 @@ impl Layout {
  dest: &str,
  ) -> CargoResult<Layout> {
  let mut root = ws.target_dir();
+ let mut build_root = ws.build_dir();
  if let Some(target) = target {
  root.push(target.short_name());
+ build_root.push(target.short_name());
  }
+ let build_dest = build_root.join(dest);
  let dest = root.join(dest);
  // If the root directory doesn't already exist go ahead and create it
  // here. Use this opportunity to exclude it from backups as well if the
  // system supports it since this is a freshly created folder.
  //
  paths::create_dir_all_excluded_from_backups_atomic(root.as_path_unlocked())?;
+ if root != build_root {
+ paths::create_dir_all_excluded_from_backups_atomic(build_root.as_path_unlocked())?;
+ }
+
  // Now that the excluded from backups target root is created we can create the
  // actual destination (sub)subdirectory.
  paths::create_dir_all(dest.as_path_unlocked())?;
@@ -167,23 +181,37 @@ impl Layout {
  // directory, so just lock the entire thing for the duration of this
  // compile.
  let lock = dest.open_rw_exclusive_create(".cargo-lock", ws.gctx(), "build directory")?;
+
+ let build_lock = if root != build_root {
+ Some(build_dest.open_rw_exclusive_create(
+ ".cargo-lock",
+ ws.gctx(),
+ "build directory",
+ )?)
+ } else {
+ None
+ };
  let root = root.into_path_unlocked();
+ let build_root = build_root.into_path_unlocked();
  let dest = dest.into_path_unlocked();
- let deps = dest.join("deps");
+ let build_dest = build_dest.as_path_unlocked();
+ let deps = build_dest.join("deps");
  let artifact = deps.join("artifact");
 
  Ok(Layout {
  deps,
- build: dest.join("build"),
+ build: build_dest.join("build"),
  artifact,
- incremental: dest.join("incremental"),
- fingerprint: dest.join(".fingerprint"),
+ incremental: build_dest.join("incremental"),
+ fingerprint: build_dest.join(".fingerprint"),
  examples: dest.join("examples"),
+ build_examples: build_dest.join("examples"),
  doc: root.join("doc"),
- tmp: root.join("tmp"),
+ tmp: build_root.join("tmp"),
  root,
  dest,
  _lock: lock,
+ _build_lock: build_lock,
  })
  }
 
@@ -193,6 +221,7 @@ impl Layout {
  paths::create_dir_all(&self.incremental)?;
  paths::create_dir_all(&self.fingerprint)?;
  paths::create_dir_all(&self.examples)?;
+ paths::create_dir_all(&self.build_examples)?;
  paths::create_dir_all(&self.build)?;
 
  Ok(())
@@ -210,6 +239,10 @@ impl Layout {
  pub fn examples(&self) -> &Path {
  &self.examples
  }
+ /// Fetch the build examples path.
+ pub fn build_examples(&self) -> &Path {
+ &self.build_examples
+ }
  /// Fetch the doc path.
  pub fn doc(&self) -> &Path {
  &self.doc