diff --git a/builder/default.nix b/builder/default.nix
index f9b14a5..55d46ac 100644
--- a/builder/default.nix
+++ b/builder/default.nix
@@ -4,7 +4,6 @@
 , buildEnv
 , lib
 , fetchgit
-, removeReferencesTo
 , jq
 , cacert
 , pkgs
@@ -227,8 +226,6 @@ let
 
       go = selectGo attrs goMod;
 
-      removeReferences = [ ] ++ optional (!allowGoReference) go;
-
       defaultPackage = modulesStruct.goPackagePath or "";
 
       vendorEnv = mkVendorEnv {
@@ -247,14 +244,14 @@ let
         } // optionalAttrs (hasAttr "subPackages" modulesStruct) {
         subPackages = modulesStruct.subPackages;
       } // attrs // {
-        nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs;
+        nativeBuildInputs = [ go ] ++ nativeBuildInputs;
 
         inherit (go) GOOS GOARCH;
 
         GO_NO_VENDOR_CHECKS = "1";
 
         GO111MODULE = "on";
-        GOFLAGS = "-mod=vendor";
+        GOFLAGS = [ "-mod=vendor" ] ++ lib.optionals (!allowGoReference) [ "-trimpath" ];
 
         configurePhase = attrs.configurePhase or ''
           runHook preConfigure
@@ -361,6 +358,9 @@ let
         checkPhase = attrs.checkPhase or ''
           runHook preCheck
 
+          # We do not set trimpath for tests, in case they reference test assets
+          export GOFLAGS=''${GOFLAGS//-trimpath/}
+
           for pkg in $(getGoDirs test); do
             buildGoDir test "$pkg"
           done
@@ -378,10 +378,6 @@ let
           runHook postInstall
         '';
 
-        preFixup = (attrs.preFixup or "") + ''
-          find $out/{bin,libexec,lib} -type f 2>/dev/null | xargs -r ${removeExpr removeReferences} || true
-        '';
-
         strictDeps = true;
 
         disallowedReferences = optional (!allowGoReference) go;