patchpkg: port ELF patching to Go and use RPATH

Move the patching functionality from glibc-patch.bash to Go while also updating it to set RPATH instead of RUNPATH.

Author: gcurtis

internal/boxcli/patch.go

@@ -6,7 +6,8 @@ import (
 )
 
 func patchCmd() *cobra.Command {
-return &cobra.Command{
+var glibc string
+cmd := &cobra.Command{
 Use: "patch <store-path>",
 Short: "Apply Devbox patches to a package to fix common linker errors",
 Args: cobra.ExactArgs(1),
@@ -16,7 +17,10 @@ func patchCmd() *cobra.Command {
 if err != nil {
 return err
 }
+builder.Glibc = glibc
 return builder.Build(cmd.Context(), args[0])
 },
 }
+cmd.Flags().StringVar(&glibc, "glibc", "", "patch binaries to use a different glibc")
+return cmd
 }

internal/patchpkg/builder.go

@@ -2,6 +2,7 @@
 package patchpkg
 
 import (
+"bufio"
 "bytes"
 "context"
 _ "embed"
@@ -12,6 +13,7 @@ import (
 "log/slog"
 "os"
 "os/exec"
+"path"
 "path/filepath"
 )
 
@@ -23,6 +25,12 @@ type DerivationBuilder struct {
 // Out is the output directory that will contain the built derivation.
 // If empty it defaults to $out, which is typically set by Nix.
 Out string
+
+// Glibc is an optional store path to an alternative glibc version. If
+// it's set, the builder will patch ELF binaries to use its shared
+// libraries and dynamic linker.
+Glibc string
+glibcPatcher glibcPatcher
 }
 
 // NewDerivationBuilder initializes a new DerivationBuilder from the current
@@ -42,6 +50,13 @@ func (d *DerivationBuilder) init() error {
 return fmt.Errorf("patchpkg: $out is empty (is this being run from a nix build?)")
 }
 }
+if d.Glibc != "" {
+var err error
+d.glibcPatcher, err = newGlibcPatcher(newPackageFS(d.Glibc))
+if err != nil {
+return fmt.Errorf("patchpkg: can't patch glibc using %s: %v", d.Glibc, err)
+}
+}
 return nil
 }
 
@@ -53,7 +68,7 @@ func (d *DerivationBuilder) Build(ctx context.Context, pkgStorePath string) erro
 }
 
 slog.DebugContext(ctx, "starting build to patch package",
-"pkg", pkgStorePath, "out", d.Out)
+"pkg", pkgStorePath, "glibc", d.Glibc, "out", d.Out)
 return d.build(ctx, newPackageFS(pkgStorePath), newPackageFS(d.Out))
 }
 
@@ -70,7 +85,7 @@ func (d *DerivationBuilder) build(ctx context.Context, pkg, out *packageFS) erro
 case isSymlink(entry.Type()):
 err = d.copySymlink(pkg, out, path)
 default:
-err = d.copyFile(pkg, out, path)
+err = d.copyFile(ctx, pkg, out, path)
 }
 
 if err != nil {
@@ -93,14 +108,28 @@ func (d *DerivationBuilder) copyDir(out *packageFS, path string) error {
 return os.Mkdir(path, 0o777)
 }
 
-func (d *DerivationBuilder) copyFile(pkg, out *packageFS, path string) error {
-src, err := pkg.Open(path)
+func (d *DerivationBuilder) copyFile(ctx context.Context, pkg, out *packageFS, path string) error {
+srcFile, err := pkg.Open(path)
 if err != nil {
 return err
 }
-defer src.Close()
+defer srcFile.Close()
 
-stat, err := src.Stat()
+src := bufio.NewReader(srcFile)
+if d.needsGlibcPatch(src, path) {
+srcPath, err := pkg.OSPath(path)
+if err != nil {
+return err
+}
+dstPath, err := out.OSPath(path)
+if err != nil {
+return err
+}
+// No need to copy the file, patchelf will do it for us.
+return d.glibcPatcher.patch(ctx, srcPath, dstPath)
+}
+
+stat, err := srcFile.Stat()
 if err != nil {
 return err
 }
@@ -142,6 +171,21 @@ func (d *DerivationBuilder) copySymlink(pkg, out *packageFS, path string) error
 return os.Symlink(target, link)
 }
 
+func (d *DerivationBuilder) needsGlibcPatch(file *bufio.Reader, filePath string) bool {
+if d.Glibc == "" {
+return false
+}
+if path.Dir(filePath) != "bin" {
+return false
+}
+
+magic, err := file.Peek(4)
+if err != nil {
+return false
+}
+return magic[0] == 0x7F && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F'
+}
+
 // packageFS is the tree of files for a package in the Nix store.
 type packageFS struct {
 fs.FS

internal/patchpkg/glibc-patch.bash

@@ -2,76 +2,21 @@
 
 set -euo pipefail
 
-declare -r pkg # package that we're patching
-declare -r glibc # new glibc that we're patching in
-declare -r out # nix output path that will contain the patched package
+declare -r pkg # package that we're patching
+declare -r out # nix output path that will contain the patched package
 
 # Paths to this script's dependencies set by nix.
-declare -r coreutils file findutils gnused patchelf ripgrep
+declare -r coreutils gnused ripgrep
 
 # Explicitly declare the specific commands that this script depends on.
-hash -p "$coreutils/bin/cp" cp
 hash -p "$coreutils/bin/chmod" chmod
 hash -p "$coreutils/bin/dirname" dirname
 hash -p "$coreutils/bin/echo" echo
-hash -p "$coreutils/bin/head" head
 hash -p "$coreutils/bin/stat" stat
 hash -p "$coreutils/bin/wc" wc
-hash -p "$file/bin/file" file
-hash -p "$findutils/bin/find" find
 hash -p "$gnused/bin/sed" sed
-hash -p "$patchelf/bin/patchelf" patchelf
 hash -p "$ripgrep/bin/rg" rg
 
-# Find the new linker that we'll patch into all of the package's executables as
-# the interpreter.
-interp="$(find "$glibc/lib" -type f -maxdepth 1 -executable -name 'ld-linux-*.so*' | head -n1)"
-readonly interp
-
-patch() {
-declare -r binary="$1" # ELF binary to patch
-
-perm=$(stat -c "%a" "$binary")
-old_rpath="$(patchelf --print-rpath "$binary")"
-new_rpath="$glibc/lib${old_rpath:+:$old_rpath}"
-
-echo "running patchelf file=\"$binary\" rpath=\"$new_rpath\" perm=\"$perm\""
-chmod u+w "$binary"
-patchelf --set-rpath "$new_rpath" \
- --add-needed libBrokenLocale.so.1 \
- --add-needed libanl.so.1 \
- --add-needed libc.so.6 \
- --add-needed libdl.so.2 \
- --add-needed libgcc_s.so.1 \
- --add-needed libm.so.6 \
- --add-needed libmvec.so.1 \
- --add-needed libnsl.so.1 \
- --add-needed libnss_compat.so.2 \
- --add-needed libnss_db.so.2 \
- --add-needed libnss_dns.so.2 \
- --add-needed libnss_files.so.2 \
- --add-needed libnss_hesiod.so.2 \
- --add-needed libpcprofile.so \
- --add-needed libpthread.so.0 \
- --add-needed libresolv.so.2 \
- --add-needed librt.so.1 \
- --add-needed libutil.so.1 \
- --set-interpreter "$interp" \
- "$binary"
-
-# Neaten the runpath by removing extraneous paths. This will likely remove any old glibc.
-patchelf --shrink-rpath "$binary"
-chmod "$perm" "$binary"
-}
-
-# Search for any files that look like ELF binaries and patch them.
-elves="$(find "$out" -type f -exec "$file/bin/file" {} \+ | rg --replace '$1' '^(.*): .*ELF.*executable.*dynamically linked.*$')"
-count="$(echo "$elves" | wc -l)"
-echo "patching elf binaries count=$count"
-for binary in $elves; do
-patch "$binary" exe
-done
-
 patch_store_path() {
 declare -r path="$1"
 declare -r perm=$(stat -c "%a" "$path")

internal/patchpkg/patch.go

@@ -0,0 +1,127 @@
+package patchpkg
+
+import (
+"bytes"
+"context"
+"fmt"
+"io/fs"
+"log/slog"
+"os/exec"
+"path"
+"slices"
+)
+
+// glibcPatcher patches ELF binaries to use an alternative version of glibc.
+type glibcPatcher struct {
+// ld is the absolute path to the new dynamic linker (ld.so).
+ld string
+
+// lib is the absolute path to the lib directory containing the new libc
+// shared objects (libc.so).
+lib string
+}
+
+// newGlibcPatcher creates a new glibcPatcher and verifies that it can find the
+// shared object files in glibc.
+func newGlibcPatcher(glibc *packageFS) (patcher glibcPatcher, err error) {
+// Verify that we can find a directory with libc in it.
+glob := "lib*/libc.so*"
+matches, _ := fs.Glob(glibc, glob)
+if len(matches) == 0 {
+return glibcPatcher{}, fmt.Errorf("cannot find libc.so file matching %q", glob)
+}
+for i := range matches {
+matches[i] = path.Dir(matches[i])
+}
+slices.Sort(matches) // pick the shortest name: lib < lib32 < lib64 < libx32
+patcher.lib, err = glibc.OSPath(matches[0])
+if err != nil {
+return glibcPatcher{}, err
+}
+slog.Debug("found new libc directory", "path", patcher.lib)
+
+// Verify that we can find the new dynamic linker.
+glob = "lib*/ld-linux*.so*"
+matches, _ = fs.Glob(glibc, glob)
+if len(matches) == 0 {
+return glibcPatcher{}, fmt.Errorf("cannot find ld.so file matching %q", glob)
+}
+slices.Sort(matches)
+patcher.ld, err = glibc.OSPath(matches[0])
+if err != nil {
+return glibcPatcher{}, err
+}
+slog.Debug("found new dynamic linker", "path", patcher.ld)
+
+return patcher, nil
+}
+
+// patch applies glibc patches to a binary and writes the patched result to
+// outPath. It does not modify the original binary in-place.
+func (g glibcPatcher) patch(ctx context.Context, path, outPath string) error {
+cmd := &patchelf{PrintInterpreter: true}
+out, err := cmd.run(ctx, path)
+if err != nil {
+return err
+}
+oldInterp := string(out)
+
+cmd = &patchelf{PrintRPATH: true}
+out, err = cmd.run(ctx, path)
+if err != nil {
+return err
+}
+oldRpath := string(out)
+
+cmd = &patchelf{
+SetInterpreter: g.ld,
+Output: outPath,
+}
+if len(oldRpath) == 0 {
+cmd.SetRPATH = g.lib
+} else {
+cmd.SetRPATH = g.lib + ":" + oldRpath
+}
+
+slog.Debug("patching glibc on binary",
+"path", path, "outPath", cmd.Output,
+"old_interp", oldInterp, "new_interp", cmd.SetInterpreter,
+"old_rpath", oldRpath, "new_rpath", cmd.SetRPATH,
+)
+_, err = cmd.run(ctx, path)
+return err
+}
+
+// patchelf runs the patchelf command.
+type patchelf struct {
+SetRPATH string
+PrintRPATH bool
+
+SetInterpreter string
+PrintInterpreter bool
+
+Output string
+}
+
+// run runs patchelf on an ELF binary and returns its output.
+func (p *patchelf) run(ctx context.Context, elf string) ([]byte, error) {
+cmd := exec.CommandContext(ctx, lookPath("patchelf"))
+if p.SetRPATH != "" {
+cmd.Args = append(cmd.Args, "--force-rpath", "--set-rpath", p.SetRPATH)
+}
+if p.PrintRPATH {
+cmd.Args = append(cmd.Args, "--print-rpath")
+}
+if p.SetInterpreter != "" {
+cmd.Args = append(cmd.Args, "--set-interpreter", p.SetInterpreter)
+}
+if p.PrintInterpreter {
+cmd.Args = append(cmd.Args, "--print-interpreter")
+}
+if p.Output != "" {
+cmd.Args = append(cmd.Args, "--output", p.Output)
+}
+cmd.Args = append(cmd.Args, elf)
+out, err := cmd.Output()
+return bytes.TrimSpace(out), err
+}

internal/shellgen/tmpl/glibc-patch.nix.tmpl

@@ -45,7 +45,7 @@
  system = pkg.system;
 
  # Programs needed by glibc-patch.bash.
- inherit (nixpkgs-glibc.legacyPackages."${system}") bash coreutils file findutils glibc gnused patchelf ripgrep;
+ inherit (nixpkgs-glibc.legacyPackages."${system}") bash coreutils glibc gnused patchelf ripgrep;
 
  # Create a package that puts the local devbox binary in the conventional
  # bin subdirectory. This also ensures that the executable is named
@@ -62,8 +62,9 @@
  args = [ "-c" "${coreutils}/bin/mkdir -p $out/bin && ${coreutils}/bin/cp ${local-devbox} $out/bin/devbox && exit 0" ];
  };
 
+ DEVBOX_DEBUG = 1;
  builder = "${devbox}/bin/devbox";
- args = [ "patch" pkg ];
+ args = [ "patch" "--glibc" glibc pkg ];
  };
  in
  {