| Serge Bazanski | 4f00f90 | 2023-12-19 13:54:04 +0100 | [diff] [blame^] | 1 | Fixes a few issues with rules_rust/rustc reproducibility when the same code is |
| 2 | being built in slightly different BuildConfigurations. |
| 3 | |
| 4 | Even if BuildConfigurations differ only by insignificant (to rules_rust) |
| 5 | configuration flags, the resulting output directory will be different (keyed by |
| 6 | an 'ST-hash' which is generated from the configuration). |
| 7 | |
| 8 | Unfortunately, rust/rules_rust really likes to embed bazel-out/<dir>/bin paths |
| 9 | into the binaries by default, thus embedding the ST-hash, thus leading to |
| 10 | different bit-for-bit binaries when built across two slightly different |
| 11 | configs. |
| 12 | |
| 13 | We fix this by doing two changes: |
| 14 | |
| 15 | 1. We override the codegen metadata hash suffix to not depend on the ST-hash |
| 16 | directory name. Otherwise, built rlibs will have a numeric .XXX suffix that |
| 17 | changes depending on the ST-hash. We have to do this separately for prost |
| 18 | codegen, too. |
| 19 | 2. We add a remap path option to rustc that replaces bazel-out/<dir>/bin/ with |
| 20 | bin/. |
| 21 | |
| 22 | diff --git a/proto/prost/private/prost.bzl b/proto/prost/private/prost.bzl |
| 23 | index 6cd3d522..56d897b5 100644 |
| 24 | --- a/proto/prost/private/prost.bzl |
| 25 | +++ b/proto/prost/private/prost.bzl |
| 26 | @@ -128,7 +128,9 @@ def _compile_rust(ctx, attr, crate_name, src, deps, edition): |
| 27 | A DepVariantInfo provider. |
| 28 | """ |
| 29 | toolchain = ctx.toolchains["@rules_rust//rust:toolchain_type"] |
| 30 | - output_hash = repr(hash(src.path + ".prost")) |
| 31 | + src_path = src.path.replace(ctx.bin_dir.path, 'bin') |
| 32 | + print(src.path, src_path) |
| 33 | + output_hash = repr(hash(src_path + ".prost")) |
| 34 | |
| 35 | lib_name = "{prefix}{name}-{lib_hash}{extension}".format( |
| 36 | prefix = "lib", |
| 37 | diff --git a/proto/protobuf/proto.bzl b/proto/protobuf/proto.bzl |
| 38 | index ca2df8a2..b4bf3335 100644 |
| 39 | --- a/proto/protobuf/proto.bzl |
| 40 | +++ b/proto/protobuf/proto.bzl |
| 41 | @@ -188,7 +188,7 @@ def _rust_proto_compile(protos, descriptor_sets, imports, crate_name, ctx, is_gr |
| 42 | srcs.append(lib_rs) |
| 43 | |
| 44 | # And simulate rust_library behavior |
| 45 | - output_hash = determine_output_hash(lib_rs, ctx.label) |
| 46 | + output_hash = determine_output_hash(ctx.bin_dir, lib_rs, ctx.label) |
| 47 | rust_lib = ctx.actions.declare_file("%s/lib%s-%s.rlib" % ( |
| 48 | output_dir, |
| 49 | crate_name, |
| 50 | diff --git a/rust/private/clippy.bzl b/rust/private/clippy.bzl |
| 51 | index 071ca29c..968a4ad6 100644 |
| 52 | --- a/rust/private/clippy.bzl |
| 53 | +++ b/rust/private/clippy.bzl |
| 54 | @@ -120,7 +120,7 @@ def _clippy_aspect_impl(target, ctx): |
| 55 | dep_info = dep_info, |
| 56 | linkstamp_outs = linkstamp_outs, |
| 57 | ambiguous_libs = ambiguous_libs, |
| 58 | - output_hash = determine_output_hash(crate_info.root, ctx.label), |
| 59 | + output_hash = determine_output_hash(ctx.bin_dir, crate_info.root, ctx.label), |
| 60 | rust_flags = [], |
| 61 | out_dir = out_dir, |
| 62 | build_env_files = build_env_files, |
| 63 | diff --git a/rust/private/rust.bzl b/rust/private/rust.bzl |
| 64 | index 014d8d0d..cb783dc9 100644 |
| 65 | --- a/rust/private/rust.bzl |
| 66 | +++ b/rust/private/rust.bzl |
| 67 | @@ -275,7 +275,7 @@ def _rust_library_common(ctx, crate_type): |
| 68 | if crate_type in ["cdylib", "staticlib"]: |
| 69 | output_hash = None |
| 70 | else: |
| 71 | - output_hash = determine_output_hash(crate_root, ctx.label) |
| 72 | + output_hash = determine_output_hash(ctx.bin_dir, crate_root, ctx.label) |
| 73 | |
| 74 | crate_name = compute_crate_name(ctx.workspace_name, ctx.label, toolchain, ctx.attr.crate_name) |
| 75 | rust_lib_name = _determine_lib_name( |
| 76 | @@ -390,7 +390,7 @@ def _rust_test_impl(ctx): |
| 77 | # Target is building the crate in `test` config |
| 78 | crate = ctx.attr.crate[rust_common.crate_info] if rust_common.crate_info in ctx.attr.crate else ctx.attr.crate[rust_common.test_crate_info].crate |
| 79 | |
| 80 | - output_hash = determine_output_hash(crate.root, ctx.label) |
| 81 | + output_hash = determine_output_hash(ctx.bin_dir, crate.root, ctx.label) |
| 82 | output = ctx.actions.declare_file( |
| 83 | "test-%s/%s%s" % ( |
| 84 | output_hash, |
| 85 | @@ -441,7 +441,7 @@ def _rust_test_impl(ctx): |
| 86 | crate_root = crate_root_src(ctx.attr.name, ctx.files.srcs, crate_root_type) |
| 87 | srcs, crate_root = _transform_sources(ctx, ctx.files.srcs, crate_root) |
| 88 | |
| 89 | - output_hash = determine_output_hash(crate_root, ctx.label) |
| 90 | + output_hash = determine_output_hash(ctx.bin_dir, crate_root, ctx.label) |
| 91 | output = ctx.actions.declare_file( |
| 92 | "test-%s/%s%s" % ( |
| 93 | output_hash, |
| 94 | diff --git a/rust/private/rustc.bzl b/rust/private/rustc.bzl |
| 95 | index e76c20bd..cebd5a73 100644 |
| 96 | --- a/rust/private/rustc.bzl |
| 97 | +++ b/rust/private/rustc.bzl |
| 98 | @@ -952,6 +952,10 @@ def construct_arguments( |
| 99 | if remap_path_prefix != None: |
| 100 | rustc_flags.add("--remap-path-prefix=${{pwd}}={}".format(remap_path_prefix)) |
| 101 | |
| 102 | + # Replace unstable bindir path (based on ST-hash which is in turn based on |
| 103 | + # build configuration) with a stable bin/ path. |
| 104 | + rustc_flags.add("--remap-path-prefix={}=bin".format(ctx.bin_dir.path)) |
| 105 | + |
| 106 | if emit: |
| 107 | rustc_flags.add_joined(emit_with_paths, format_joined = "--emit=%s", join_with = ",") |
| 108 | if error_format != "json": |
| 109 | diff --git a/rust/private/utils.bzl b/rust/private/utils.bzl |
| 110 | index 879e2b62..08ef530f 100644 |
| 111 | --- a/rust/private/utils.bzl |
| 112 | +++ b/rust/private/utils.bzl |
| 113 | @@ -181,7 +181,7 @@ def abs(value): |
| 114 | return -value |
| 115 | return value |
| 116 | |
| 117 | -def determine_output_hash(crate_root, label): |
| 118 | +def determine_output_hash(bin_dir, crate_root, label): |
| 119 | """Generates a hash of the crate root file's path. |
| 120 | |
| 121 | Args: |
| 122 | @@ -192,8 +192,11 @@ def determine_output_hash(crate_root, label): |
| 123 | str: A string representation of the hash. |
| 124 | """ |
| 125 | |
| 126 | + # Remove any unstable BuildConfiguration derived dir fragments to unify |
| 127 | + # hashes between different configs. |
| 128 | + crate_root_path = crate_root.path.replace(bin_dir.path, 'bin') |
| 129 | # Take the absolute value of hash() since it could be negative. |
| 130 | - h = abs(hash(crate_root.path) + hash(repr(label))) |
| 131 | + h = abs(hash(crate_root_path) + hash(repr(label))) |
| 132 | return repr(h) |
| 133 | |
| 134 | def get_preferred_artifact(library_to_link, use_pic): |