refactor(zipapp): port zipper to Rust and add it to the shrubbery toolchain (#4407) Replaces the Python zipper used by py_zipapp's PyZipAppCreateZip action with a Rust implementation, so creating zipapps no longer requires a Python interpreter in the exec configuration. The crates/zipper library mirrors tools/zipapp/zipper.py: same manifest format, symlink-precedence sorting, sandboxed source symlink detection, fixed 1980-01-01 timestamps, and permission preservation. It is exposed as the `zipper` shrubbery subcommand and selected via get_shrubbery_tool, falling back to the Python tool when shrubbery is unavailable. The `zip` crate is pinned to 8.x: 9.x adds target-conditional OS deps that fail on powerpc64le until rules_rust supports that platform (see the NOTE in MODULE.bazel). Error handling uses `anyhow` so failures name the manifest line or the path they concern. Rust code is linted in CI by a new `rust_lint` BazelCI task (`bazel build --config=rust-lint //...`), which runs clippy with warnings as errors and `unwrap()` forbidden, checks rustfmt, and denies `expect()` in the non-test crates. ## Performance Zip creation time only (x86_64 Linux, `-c opt`; median of 5 runs for the small cases, single run for the large ones): ``` Workload Comp Python Rust Ratio 3k .py + 5x8 MB blobs (46 MB) 0 0.43s 0.26s 1.6x same 6 1.48s 1.59s 0.9x 10k small .py files 0 1.19s 0.70s 1.7x same 6 1.42s 1.07s 1.3x CPython runtime + 10k .py (405 MB) 0 4.08s 1.19s 3.4x + torch/CUDA wheels (7.3 GB, ZIP64) 0 9.7s 7.9s 1.2x + torch/CUDA wheels 6 335s 370s 0.9x ``` Rust is faster on per-entry overhead, which dominates the default uncompressed case. Deflate throughput is ~10% slower because `flate2` uses the pure-Rust miniz_oxide backend (chosen so the tool cross-compiles for all `dev/platforms`); CPython's `zipfile` uses C zlib.
diff --git a/.bazelrc b/.bazelrc index a5b9664..ed25451 100644 --- a/.bazelrc +++ b/.bazelrc
@@ -23,6 +23,17 @@ build:pyrefly --aspects=//tests/support/pyrefly:pyrefly.bzl%pyrefly_aspect build:pyrefly --output_groups=+pyrefly +# Lint Rust code with clippy and check rustfmt formatting on every build that +# touches a Rust target, so problems surface locally before CI. The output +# groups are additive (`+`), so normal build outputs are unaffected. +build --config=rust-lint +build:rust-lint --aspects=@rules_rust//rust:defs.bzl%rust_clippy_aspect +build:rust-lint --aspects=@rules_rust//rust:defs.bzl%rustfmt_aspect +build:rust-lint --output_groups=+clippy_checks,+rustfmt_checks +# `unwrap()` is forbidden everywhere; the non-test crates additionally deny +# `expect()` via a crate-level attribute. +build:rust-lint --@rules_rust//rust/settings:clippy_flags=-Dwarnings,-Dclippy::unwrap_used + # Ensure ongoing compatibility with this flag. common --incompatible_disallow_struct_provider_syntax # Makes Bazel 7 act more like Bazel 8
diff --git a/.bazelrc.deleted_packages b/.bazelrc.deleted_packages index 4b8b5bf..e17e1f2 100644 --- a/.bazelrc.deleted_packages +++ b/.bazelrc.deleted_packages
@@ -57,3 +57,4 @@ common --deleted_packages=tests/modules/other/with_external_data common --deleted_packages=tests/modules/rules_pyrefly_stub/pyrefly common --deleted_packages=tests/modules/rules_rust_stub/rust +common --deleted_packages=tests/modules/rules_rust_stub/rust/settings
diff --git a/MODULE.bazel b/MODULE.bazel index 6ebc202..ab53d60 100644 --- a/MODULE.bazel +++ b/MODULE.bazel
@@ -112,7 +112,22 @@ dev_dependency = TOOL_DEPS_DEV_ONLY, ) -# keep-sorted start block=yes +# keep-sorted start block=yes by_regex=package\s*=\s*"([^"]+)" newline_separated=yes +rust_crates.spec( + package = "anyhow", + version = "1.0.104", +) + +# Not used directly; it's a transitive dep of zip. zip's `deflate-flate2` +# feature enables flate2 without selecting a backend, so a top-level spec is +# needed to pick the pure-Rust `rust_backend` and avoid the C zlib backend, +# which can't cross-compile for dev/platforms. +rust_crates.spec( + default_features = False, + features = ["rust_backend"], + package = "flate2", + version = "1.1.10", +) rust_crates.spec( package = "lexopt", version = "0.3.2", @@ -121,6 +136,17 @@ package = "sha2", version = "0.10.8", ) + +# NOTE: zip 9.x added mandatory target-conditional OS deps (getrandom, nix, +# windows-sys) that fail on powerpc64le-unknown-linux-musl until rules_rust +# defines a powerpc64le @rules_rust//rust/platform config_setting. Keep on the +# latest 8.x release until rules_rust supports powerpc64le platform selection. +rust_crates.spec( + default_features = False, + features = ["deflate-flate2"], + package = "zip", + version = "8.6.0", +) # keep-sorted end # The crates must build for every platform in dev/platforms. Conditional @@ -147,7 +173,7 @@ # rules_rust has no @rules_rust//rust/platform config_setting for Linux ppc64le, # so don't mark the crates as incompatible with platforms that don't match one # of the triples above. Such platforms get only the unconditional dependencies, -# which suffice for lexopt and sha2. +# which suffice for anyhow, flate2, lexopt, sha2, and zip. rust_crates.render_config( generate_target_compatible_with = False, )
diff --git a/crates/Cargo.Bazel.lock b/crates/Cargo.Bazel.lock index c7dbb08..1f3a917 100644 --- a/crates/Cargo.Bazel.lock +++ b/crates/Cargo.Bazel.lock
@@ -1,6 +1,121 @@ { - "checksum": "30f93d6303d71ee7044f0ee40523812209c2025ff5210bb2c7008ee2eee9782a", + "checksum": "7f9a06aa6df743d20963095d0fe2ee50cbe1bac5ffd344c8b29c7e2851576f07", "crates": { + "adler2 2.0.1": { + "name": "adler2", + "version": "2.0.1", + "package_url": "https://github.com/oyvindln/adler2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/adler2/2.0.1/download", + "sha256": "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + } + }, + "targets": [ + { + "Library": { + "crate_name": "adler2", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "adler2", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2021", + "version": "2.0.1" + }, + "license": "0BSD OR MIT OR Apache-2.0", + "license_ids": [ + "0BSD", + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-0BSD" + }, + "anyhow 1.0.104": { + "name": "anyhow", + "version": "1.0.104", + "package_url": "https://github.com/dtolnay/anyhow", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/anyhow/1.0.104/download", + "sha256": "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + } + }, + "targets": [ + { + "Library": { + "crate_name": "anyhow", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "anyhow", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "extra_deps": { + "common": [ + ":build_script_build" + ], + "selects": {} + }, + "edition": "2021", + "version": "1.0.104" + }, + "build_script_attrs": { + "compile_data_glob": [ + "**" + ], + "compile_data_glob_excludes": [ + "**/*.rs" + ], + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "block-buffer 0.10.4": { "name": "block-buffer", "version": "0.10.4", @@ -156,6 +271,90 @@ ], "license_file": "LICENSE-APACHE" }, + "crc32fast 1.5.4": { + "name": "crc32fast", + "version": "1.5.4", + "package_url": "https://github.com/srijs/rust-crc32fast", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/crc32fast/1.5.4/download", + "sha256": "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" + } + }, + "targets": [ + { + "Library": { + "crate_name": "crc32fast", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + }, + { + "BuildScript": { + "crate_name": "build_script_build", + "crate_root": "build.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "crc32fast", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "cfg-if 1.0.5", + "target": "cfg_if" + } + ], + "selects": {} + }, + "extra_deps": { + "common": [ + ":build_script_build" + ], + "selects": {} + }, + "edition": "2021", + "version": "1.5.4" + }, + "build_script_attrs": { + "compile_data_glob": [ + "**" + ], + "compile_data_glob_excludes": [ + "**/*.rs" + ], + "data_glob": [ + "**" + ] + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "crypto-common 0.1.7": { "name": "crypto-common", "version": "0.1.7", @@ -303,12 +502,24 @@ "deps": { "common": [ { + "id": "anyhow 1.0.104", + "target": "anyhow" + }, + { + "id": "flate2 1.1.11", + "target": "flate2" + }, + { "id": "lexopt 0.3.2", "target": "lexopt" }, { "id": "sha2 0.10.9", "target": "sha2" + }, + { + "id": "zip 8.6.0", + "target": "zip" } ], "selects": {} @@ -320,6 +531,105 @@ "license_ids": [], "license_file": null }, + "equivalent 1.0.3": { + "name": "equivalent", + "version": "1.0.3", + "package_url": "https://github.com/indexmap-rs/equivalent", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/equivalent/1.0.3/download", + "sha256": "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + } + }, + "targets": [ + { + "Library": { + "crate_name": "equivalent", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "equivalent", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2015", + "version": "1.0.3" + }, + "license": "Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, + "flate2 1.1.11": { + "name": "flate2", + "version": "1.1.11", + "package_url": "https://github.com/rust-lang/flate2-rs", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/flate2/1.1.11/download", + "sha256": "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" + } + }, + "targets": [ + { + "Library": { + "crate_name": "flate2", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "flate2", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "any_impl", + "miniz_oxide", + "rust_backend" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "crc32fast 1.5.4", + "target": "crc32fast" + }, + { + "id": "miniz_oxide 0.9.1", + "target": "miniz_oxide" + } + ], + "selects": {} + }, + "edition": "2018", + "version": "1.1.11" + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "generic-array 0.14.7": { "name": "generic-array", "version": "0.14.7", @@ -411,6 +721,104 @@ ], "license_file": "LICENSE" }, + "hashbrown 0.17.1": { + "name": "hashbrown", + "version": "0.17.1", + "package_url": "https://github.com/rust-lang/hashbrown", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/hashbrown/0.17.1/download", + "sha256": "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + } + }, + "targets": [ + { + "Library": { + "crate_name": "hashbrown", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "hashbrown", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2024", + "version": "0.17.1" + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, + "indexmap 2.14.2": { + "name": "indexmap", + "version": "2.14.2", + "package_url": "https://github.com/indexmap-rs/indexmap", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/indexmap/2.14.2/download", + "sha256": "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" + } + }, + "targets": [ + { + "Library": { + "crate_name": "indexmap", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "indexmap", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "equivalent 1.0.3", + "target": "equivalent" + }, + { + "id": "hashbrown 0.17.1", + "target": "hashbrown" + } + ], + "selects": {} + }, + "edition": "2024", + "version": "2.14.2" + }, + "license": "Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "lexopt 0.3.2": { "name": "lexopt", "version": "0.3.2", @@ -517,6 +925,115 @@ ], "license_file": "LICENSE-APACHE" }, + "memchr 2.8.3": { + "name": "memchr", + "version": "2.8.3", + "package_url": "https://github.com/BurntSushi/memchr", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/memchr/2.8.3/download", + "sha256": "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + } + }, + "targets": [ + { + "Library": { + "crate_name": "memchr", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "memchr", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "alloc", + "default", + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "2.8.3" + }, + "license": "Unlicense OR MIT", + "license_ids": [ + "MIT", + "Unlicense" + ], + "license_file": "LICENSE-MIT" + }, + "miniz_oxide 0.9.1": { + "name": "miniz_oxide", + "version": "0.9.1", + "package_url": "https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/miniz_oxide/0.9.1/download", + "sha256": "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "miniz_oxide", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "miniz_oxide", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "simd", + "simd-adler32", + "with-alloc" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "adler2 2.0.1", + "target": "adler2" + }, + { + "id": "simd-adler32 0.3.10", + "target": "simd_adler32" + } + ], + "selects": {} + }, + "edition": "2021", + "version": "0.9.1" + }, + "license": "MIT OR Zlib OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT", + "Zlib" + ], + "license_file": "LICENSE" + }, "sha2 0.10.9": { "name": "sha2", "version": "0.10.9", @@ -583,6 +1100,90 @@ ], "license_file": "LICENSE-APACHE" }, + "simd-adler32 0.3.10": { + "name": "simd-adler32", + "version": "0.3.10", + "package_url": "https://github.com/mcountryman/simd-adler32", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/simd-adler32/0.3.10/download", + "sha256": "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + } + }, + "targets": [ + { + "Library": { + "crate_name": "simd_adler32", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "simd_adler32", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "edition": "2018", + "version": "0.3.10" + }, + "license": "MIT", + "license_ids": [ + "MIT" + ], + "license_file": "LICENSE.md" + }, + "typed-path 0.12.4": { + "name": "typed-path", + "version": "0.12.4", + "package_url": "https://github.com/chipsenkbeil/typed-path", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/typed-path/0.12.4/download", + "sha256": "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c" + } + }, + "targets": [ + { + "Library": { + "crate_name": "typed_path", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "typed_path", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "default", + "std" + ], + "selects": {} + }, + "edition": "2021", + "version": "0.12.4" + }, + "license": "MIT OR Apache-2.0", + "license_ids": [ + "Apache-2.0", + "MIT" + ], + "license_file": "LICENSE-APACHE" + }, "typenum 1.20.1": { "name": "typenum", "version": "1.20.1", @@ -660,6 +1261,76 @@ "MIT" ], "license_file": "LICENSE-APACHE" + }, + "zip 8.6.0": { + "name": "zip", + "version": "8.6.0", + "package_url": "https://github.com/zip-rs/zip2", + "repository": { + "Http": { + "url": "https://static.crates.io/crates/zip/8.6.0/download", + "sha256": "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" + } + }, + "targets": [ + { + "Library": { + "crate_name": "zip", + "crate_root": "src/lib.rs", + "srcs": { + "allow_empty": true, + "include": [ + "**/*.rs" + ] + } + } + } + ], + "library_target_name": "zip", + "common_attrs": { + "compile_data_glob": [ + "**" + ], + "crate_features": { + "common": [ + "_deflate-any", + "deflate-flate2" + ], + "selects": {} + }, + "deps": { + "common": [ + { + "id": "crc32fast 1.5.4", + "target": "crc32fast" + }, + { + "id": "flate2 1.1.11", + "target": "flate2" + }, + { + "id": "indexmap 2.14.2", + "target": "indexmap" + }, + { + "id": "memchr 2.8.3", + "target": "memchr" + }, + { + "id": "typed-path 0.12.4", + "target": "typed_path" + } + ], + "selects": {} + }, + "edition": "2024", + "version": "8.6.0" + }, + "license": "MIT", + "license_ids": [ + "MIT" + ], + "license_file": "LICENSE" } }, "binary_crates": [], @@ -717,8 +1388,11 @@ ] }, "direct_deps": [ + "anyhow 1.0.104", + "flate2 1.1.11", "lexopt 0.3.2", - "sha2 0.10.9" + "sha2 0.10.9", + "zip 8.6.0" ], "direct_dev_deps": [], "unused_patches": []
diff --git a/crates/Cargo.lock b/crates/Cargo.lock index 05055d4..4454b52 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock
@@ -3,6 +3,18 @@ version = 4 [[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] name = "block-buffer" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -27,6 +39,15 @@ ] [[package]] +name = "crc32fast" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +dependencies = [ + "cfg-if", +] + +[[package]] name = "crypto-common" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -50,8 +71,27 @@ name = "direct-cargo-bazel-deps" version = "0.0.1" dependencies = [ + "anyhow", + "flate2", "lexopt", "sha2", + "zip", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide", ] [[package]] @@ -65,6 +105,22 @@ ] [[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] name = "lexopt" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -77,6 +133,22 @@ checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] name = "sha2" version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -88,6 +160,18 @@ ] [[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "typed-path" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c" + +[[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -98,3 +182,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "typed-path", +]
diff --git a/crates/exe_zip_maker/src/lib.rs b/crates/exe_zip_maker/src/lib.rs index 936cc96..17fe2c4 100644 --- a/crates/exe_zip_maker/src/lib.rs +++ b/crates/exe_zip_maker/src/lib.rs
@@ -1,5 +1,8 @@ //! Library supporting creating self-executable zip files. +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + use std::ffi::OsString; use std::fs::{self, File}; use std::io::{self, BufReader, BufWriter, Read, Write};
diff --git a/crates/shrubbery/BUILD.bazel b/crates/shrubbery/BUILD.bazel index 338dff3..9d3dd1e 100644 --- a/crates/shrubbery/BUILD.bazel +++ b/crates/shrubbery/BUILD.bazel
@@ -11,6 +11,7 @@ edition = "2021", deps = [ "//crates/exe_zip_maker:exe_zip_maker_lib", + "//crates/zipper:zipper_lib", ], )
diff --git a/crates/shrubbery/src/lib.rs b/crates/shrubbery/src/lib.rs index 7b06fda..ef97e45 100644 --- a/crates/shrubbery/src/lib.rs +++ b/crates/shrubbery/src/lib.rs
@@ -1,12 +1,16 @@ //! Unified `shrubbery` ("Shell replacements for Bazel build tools in Rust") subcommand //! dispatcher for `@rules_python` Rust build tools. +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + use std::ffi::OsString; /// Canonical subcommand names supported by `shrubbery`. pub const SUBCOMMANDS: &[&str] = &[ // keep-sorted start "exe_zip_maker", + "zipper", // keep-sorted end ]; @@ -47,7 +51,10 @@ } match sub_raw.as_ref() { + // keep-sorted start "exe_zip_maker" => exe_zip_maker_lib::run_cli(iter), + "zipper" => zipper_lib::run_cli(iter), + // keep-sorted end _ => { eprintln!("shrubbery: error: unknown subcommand '{}'", sub_raw); print_usage(&prog_name, true);
diff --git a/crates/shrubbery/src/main.rs b/crates/shrubbery/src/main.rs index 425eb4c..5399c67 100644 --- a/crates/shrubbery/src/main.rs +++ b/crates/shrubbery/src/main.rs
@@ -1,3 +1,6 @@ +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + use std::env; use std::process;
diff --git a/crates/zipper/BUILD.bazel b/crates/zipper/BUILD.bazel new file mode 100644 index 0000000..1c925c8 --- /dev/null +++ b/crates/zipper/BUILD.bazel
@@ -0,0 +1,21 @@ +load("@rules_rust//rust:defs.bzl", "rust_library") +load("//python/private:distribution_filegroup.bzl", "distribution_filegroup") # buildifier: disable=bzl-visibility + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_library( + name = "zipper_lib", + srcs = ["src/lib.rs"], + edition = "2021", + deps = [ + "@crates//:anyhow", + "@crates//:lexopt", + "@crates//:zip", + ], +) + +distribution_filegroup( + name = "distribution", +)
diff --git a/crates/zipper/src/lib.rs b/crates/zipper/src/lib.rs new file mode 100644 index 0000000..05e0dd3 --- /dev/null +++ b/crates/zipper/src/lib.rs
@@ -0,0 +1,625 @@ +//! Library supporting creating deterministic zipapp archives from a manifest. + +// Errors must be returned, not panicked; see the `rust-lint` config in .bazelrc. +#![deny(clippy::expect_used, clippy::unwrap_used)] + +use std::collections::HashSet; +use std::ffi::OsString; +use std::fs::{self, File, Metadata}; +use std::io::{self, BufWriter, Seek, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{bail, Context, Result}; +use zip::write::SimpleFileOptions; +use zip::{CompressionMethod, DateTime, ZipWriter, ZIP64_BYTES_THR}; + +/// `BufWriter` capacity for the output zip. +/// +/// 256 KiB matches the other zipapp tools (`exe_zip_maker`, `zip_main_maker.py`). It is large +/// enough that the many small entries of a typical zipapp coalesce into few `write` syscalls, +/// while staying well below sizes where a bigger buffer stops paying off. +const BLOCK_SIZE: usize = 256 * 1024; + +/// Usage text for `--help`. `lexopt` does not generate help, so this is maintained by hand. +const USAGE: &str = "\ +Usage: zipper [OPTIONS] <manifest> <output> + +Create a zip file from a manifest. + +Arguments: + <manifest> Path to the manifest file. Line formats are documented on + `parse_entry` in crates/zipper/src/lib.rs. + <output> Path to the output zip file + +Options: + --compression <N> Compression level (0 for stored, 1-9 for deflated) [default: 0] + --workspace-name <NAME> Name of the workspace [default: \"\"] + --legacy-external-runfiles <0|1> Whether to use legacy external runfiles behavior [default: 0] + --runfiles-dir <DIR> Name of the runfiles directory [default: runfiles] + --target-platform-pathsep <SEP> Path separator for the target platform [default: host separator] + -h, --help Print this help"; + +/// Joins path components with `/`. +fn unix_join(parts: &[&str]) -> String { + parts.join("/") +} + +/// Computes the zip path (relative to the zip root) for a runfile entry. +fn get_zip_runfiles_path( + path: &str, + workspace_name: &str, + legacy_external_runfiles: bool, + runfiles_dir: &str, +) -> String { + if legacy_external_runfiles { + if let Some(rest) = path.strip_prefix("external/") { + return unix_join(&[runfiles_dir, rest]); + } + } + if let Some(rest) = path.strip_prefix("../") { + return unix_join(&[runfiles_dir, rest]); + } + unix_join(&[runfiles_dir, workspace_name, path]) +} + +/// Splits `path` into its components, dropping empty components and `.`, and resolving `..` +/// against the preceding component where possible (a leading run of `..` is kept, since there +/// is nothing to cancel it against). +/// +/// Both `/` and `\` are treated as separators so the result is the same on every host. +/// For example, `a/./b\c/../d` becomes `["a", "b", "d"]`, and `../x` becomes `["..", "x"]`. +fn normalize_components(path: &str) -> Vec<&str> { + let mut out = Vec::new(); + for part in path.split(['/', '\\']) { + if part.is_empty() || part == "." { + continue; + } + if part == ".." { + if let Some(&last) = out.last() { + if last != ".." { + out.pop(); + continue; + } + } + out.push(".."); + } else { + out.push(part); + } + } + out +} + +/// Computes a relative path from `start` directory to `target` using `/` separators, +/// matching `os.path.relpath(target, start=start)`. +/// +/// This is hand-rolled rather than using `std::path` because `Path` semantics depend on the +/// host OS (e.g. `\` is not a separator on Unix, and output uses `\` on Windows), whereas zip +/// entry paths must be computed identically on every host. +pub fn relative_path(target: &str, start: &str) -> String { + let start_parts = normalize_components(start); + let target_parts = normalize_components(target); + let common = start_parts + .iter() + .zip(target_parts.iter()) + .take_while(|(a, b)| a == b) + .count(); + + let mut result = vec![".."; start_parts.len() - common]; + result.extend_from_slice(&target_parts[common..]); + if result.is_empty() { + ".".to_string() + } else { + result.join("/") + } +} + +/// Converts a symlink target path to the target-platform format. +/// +/// On Windows, relative symlinks must use backslashes. +fn convert_symlink_target(path: &str, platform_pathsep: &str) -> String { + if platform_pathsep == "/" { + // Convert Windows to Unix + path.replace('\\', "/") + } else { + // Convert Unix to Windows + path.replace('/', "\\") + } +} + +/// Normalizes zip entry paths (relative to the zip root) to use forward slashes. +/// +/// Zip files use forward slash for the entries, even on Windows. +fn normalize_zip_path(path: &str) -> String { + path.replace('\\', "/") +} + +/// Parses the `is_symlink` field of a manifest line. +fn parse_symlink_mode(value: &str) -> Result<SymlinkMode> { + match value { + "0" => Ok(SymlinkMode::Regular), + "1" => Ok(SymlinkMode::Symlink), + "-1" => Ok(SymlinkMode::Infer), + _ => bail!("invalid is_symlink value (expected 0, 1, or -1): {value}"), + } +} + +/// Parses a single line of a manifest file. Returns `None` for blank lines. +/// +/// Lines have one of the following `|`-separated formats: +/// +/// * `regular|is_symlink|zip_path|content_path`: Stores `zip_path` at the exact +/// zip-root-relative path given, reading its content (or symlink target) from `content_path`. +/// * `rf-empty|runfile_path`: A `runfiles.empty_filenames` entry (such as an implicit +/// `__init__.py`) stored as an empty file under `<runfiles_dir>/<runfiles_root_rel_path>`. +/// * `rf-file|is_symlink|runfile_path|content_path`: A `runfiles.files` entry whose destination +/// zip path is computed under `<runfiles_dir>` from `runfile_path` (accounting for +/// `workspace_name` and `legacy_external_runfiles`), reading content from `content_path`. +/// * `rf-symlink|is_symlink|runfile_symlink_path|content_path`: A `runfiles.symlinks` entry +/// stored under `<runfiles_dir>/<workspace_name>/<runfile_symlink_path>`, reading content +/// from `content_path`. +/// * `rf-root-symlink|is_symlink|runfile_root_path|content_path`: A `runfiles.root_symlinks` +/// entry stored under `<runfiles_dir>/<runfile_root_path>`, reading content from +/// `content_path`. +/// * `symlink|runfile_root_path|link_to_rf_path`: An explicit symlink entry stored at +/// `<runfiles_dir>/<runfile_root_path>` whose symlink target is computed as the relative path +/// from its parent directory to `<runfiles_dir>/<link_to_rf_path>`. Takes precedence over +/// non-symlink entries at the same `zip_path`. +/// +/// See [`SymlinkMode`] for the valid `is_symlink` values. +fn parse_entry( + line: &str, + workspace_name: &str, + legacy_external_runfiles: bool, + runfiles_dir: &str, +) -> Result<Option<ManifestEntry>> { + let line = line.trim(); + if line.is_empty() { + return Ok(None); + } + + // Closures (not values): `file(...)` builds a `File` entry and `rf_path(...)` computes a + // runfiles zip path. They capture the function arguments so the match arms stay short. + let file = |content_path: &str, is_symlink: &str| -> Result<EntryKind> { + Ok(EntryKind::File { + content_path: PathBuf::from(content_path), + symlink_mode: parse_symlink_mode(is_symlink)?, + }) + }; + let rf_path = |runfile_path: &str| { + get_zip_runfiles_path( + runfile_path, + workspace_name, + legacy_external_runfiles, + runfiles_dir, + ) + }; + + let parts: Vec<&str> = line.split('|').collect(); + let entry = match parts.as_slice() { + ["regular", is_symlink, zip_path, content_path] => ManifestEntry { + zip_path: zip_path.to_string(), + kind: file(content_path, is_symlink)?, + }, + ["rf-empty", runfile_path] => ManifestEntry { + zip_path: rf_path(runfile_path), + kind: EntryKind::Empty, + }, + ["rf-file", is_symlink, runfile_path, content_path] => ManifestEntry { + zip_path: rf_path(runfile_path), + kind: file(content_path, is_symlink)?, + }, + ["rf-symlink", is_symlink, runfile_path, content_path] => ManifestEntry { + zip_path: unix_join(&[runfiles_dir, workspace_name, runfile_path]), + kind: file(content_path, is_symlink)?, + }, + ["rf-root-symlink", is_symlink, runfile_path, content_path] => ManifestEntry { + zip_path: unix_join(&[runfiles_dir, runfile_path]), + kind: file(content_path, is_symlink)?, + }, + ["symlink", runfile_path, link_to_rf_path] => { + let zip_path = unix_join(&[runfiles_dir, runfile_path]); + let link_to = unix_join(&[runfiles_dir, link_to_rf_path]); + let parent = zip_path.rsplit_once('/').map_or("", |(dir, _)| dir); + let target = relative_path(&link_to, parent); + ManifestEntry { + zip_path, + kind: EntryKind::Symlink { target }, + } + } + _ => bail!("unknown entry type or invalid format: {line}"), + }; + + Ok(Some(entry)) +} + +fn sort_key(entry: &ManifestEntry) -> (&str, bool) { + // Sort by zip path, then symlink entries first so they have precedence. + ( + &entry.zip_path, + !matches!(entry.kind, EntryKind::Symlink { .. }), + ) +} + +/// Reads and sorts all entries from the manifest file at `manifest_path`. +fn read_manifest(options: &ZipOptions) -> Result<Vec<ManifestEntry>> { + let content = fs::read_to_string(&options.manifest_path) + .with_context(|| format!("reading manifest {}", options.manifest_path.display()))?; + let mut entries = Vec::new(); + for (line_idx, line) in content.lines().enumerate() { + let parsed = parse_entry( + line, + &options.workspace_name, + options.legacy_external_runfiles, + &options.runfiles_dir, + ) + .with_context(|| format!("manifest line {}", line_idx + 1))?; + if let Some(entry) = parsed { + entries.push(entry); + } + } + entries.sort_by(|a, b| sort_key(a).cmp(&sort_key(b))); + Ok(entries) +} + +fn is_symlink_path(path: &Path) -> bool { + fs::symlink_metadata(path) + .map(|m| m.file_type().is_symlink()) + .unwrap_or(false) +} + +fn is_abs_path(path: &Path) -> bool { + if path.is_absolute() { + return true; + } + // On Windows, `is_absolute()` requires a drive prefix, so rooted-but-driveless paths + // (`\foo`, `/foo`) are reported as relative. Such paths still only resolve on the build + // machine, so treat them as absolute too. + let s = path.to_string_lossy(); + s.starts_with('/') || s.starts_with('\\') +} + +fn read_link_string(path: &Path) -> Result<String> { + let target = + fs::read_link(path).with_context(|| format!("reading symlink {}", path.display()))?; + Ok(target.to_string_lossy().into_owned()) +} + +/// Determines whether `content_path` should be stored as a symlink and returns its target +/// (relative to the symlink's parent directory). +fn source_symlink_target(content_path: &Path, symlink_mode: SymlinkMode) -> Result<Option<String>> { + let symlink_mode = match symlink_mode { + SymlinkMode::Infer if fs::metadata(content_path).is_err() => SymlinkMode::Symlink, + SymlinkMode::Infer => SymlinkMode::Regular, + mode => mode, + }; + if symlink_mode == SymlinkMode::Symlink { + return Ok(Some(read_link_string(content_path)?)); + } + if !is_symlink_path(content_path) { + return Ok(None); + } + + let target = fs::read_link(content_path) + .with_context(|| format!("reading symlink {}", content_path.display()))?; + if !is_abs_path(&target) { + return Ok(Some(target.to_string_lossy().into_owned())); + } + + // Bazel sandboxes expose regular inputs as absolute symlinks. Look through + // that indirection to detect whether the original source is also a symlink. + // Only a relative link can be kept: an absolute one points into this + // machine's Bazel output base (e.g. the _solib links Bazel creates for shared + // libraries) and would dangle wherever the zipapp runs, so store the file. + if is_symlink_path(&target) { + let source_target = fs::read_link(&target) + .with_context(|| format!("reading symlink {}", target.display()))?; + if !is_abs_path(&source_target) { + return Ok(Some(source_target.to_string_lossy().into_owned())); + } + } + Ok(None) +} + +/// Returns the Unix permission bits (`0o777` mask) to store for a file. +/// +/// The zip crate's `unix_permissions()` keeps only the permission bits and sets the file type +/// itself, so special bits (setuid, setgid, sticky) are not preserved. +fn mode_from_metadata(meta: &Metadata) -> u32 { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + meta.permissions().mode() + } + // `not(unix)` (rather than `windows`) so every target has a definition. + #[cfg(not(unix))] + { + // Mirrors what Python's `os.stat` reports on Windows. + if meta.permissions().readonly() { + 0o444 + } else { + 0o666 + } + } +} + +/// Fixed timestamp for all entries so output is deterministic. This is the earliest +/// MS-DOS timestamp a zip can represent (1980-01-01 00:00:00) and matches the Python +/// implementation. +const FIXED_TIMESTAMP: DateTime = DateTime::DEFAULT; + +fn file_options(compress_level: u32, unix_mode: u32) -> SimpleFileOptions { + let options = SimpleFileOptions::default() + .last_modified_time(FIXED_TIMESTAMP) + .unix_permissions(unix_mode); + if compress_level == 0 { + options.compression_method(CompressionMethod::Stored) + } else { + options + .compression_method(CompressionMethod::Deflated) + .compression_level(Some(i64::from(compress_level))) + } +} + +/// Options for symlink entries. `add_symlink` already sets `0o777 | S_IFLNK` and always stores +/// symlinks uncompressed, so only the timestamp needs fixing. +fn symlink_options() -> SimpleFileOptions { + SimpleFileOptions::default().last_modified_time(FIXED_TIMESTAMP) +} + +fn write_entry<W: Write + Seek>( + zf: &mut ZipWriter<W>, + entry: &ManifestEntry, + options: &ZipOptions, + seen: &mut HashSet<String>, +) -> Result<()> { + // Normalize slashes, otherwise the `seen` logic doesn't work correctly. + let zip_path = normalize_zip_path(&entry.zip_path); + let adding = || format!("adding {zip_path}"); + if !seen.insert(zip_path.clone()) { + // This can occur because symlink entries have precedence over non-symlink entries. + return Ok(()); + } + + let (content_path, symlink_mode) = match &entry.kind { + EntryKind::Empty => { + // Create empty file + zf.start_file(&zip_path, file_options(options.compress_level, 0o644)) + .with_context(adding)?; + return Ok(()); + } + EntryKind::Symlink { target } => { + let target = convert_symlink_target(target, &options.platform_pathsep); + zf.add_symlink(&zip_path, target, symlink_options()) + .with_context(adding)?; + return Ok(()); + } + EntryKind::File { + content_path, + symlink_mode, + } => (content_path, *symlink_mode), + }; + + if let Some(symlink_target) = source_symlink_target(content_path, symlink_mode)? { + let target = convert_symlink_target(&symlink_target, &options.platform_pathsep); + zf.add_symlink(&zip_path, target, symlink_options()) + .with_context(adding)?; + return Ok(()); + } + + let mut src = + File::open(content_path).with_context(|| format!("opening {}", content_path.display()))?; + let meta = src + .metadata() + .with_context(|| format!("reading metadata of {}", content_path.display()))?; + // Preserve permissions, otherwise execute is dropped. + let mode = mode_from_metadata(&meta); + let opts = file_options(options.compress_level, mode).large_file(meta.len() >= ZIP64_BYTES_THR); + zf.start_file(&zip_path, opts).with_context(adding)?; + io::copy(&mut src, zf) + .with_context(|| format!("copying {} into {zip_path}", content_path.display()))?; + Ok(()) +} + +/// Creates a zip file from a manifest according to `options`. +pub fn create_zip(options: &ZipOptions) -> Result<()> { + if options.compress_level > 9 { + bail!( + "invalid compress_level (expected 0-9): {}", + options.compress_level + ); + } + let entries = read_manifest(options)?; + + if let Some(parent) = options.output_zip.parent() { + // TODO: Use `Path::is_empty()` once it is stable (Rust 1.98). + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent) + .with_context(|| format!("creating directory {}", parent.display()))?; + } + } + + let out_file = File::create(&options.output_zip) + .with_context(|| format!("creating {}", options.output_zip.display()))?; + let out_file = BufWriter::with_capacity(BLOCK_SIZE, out_file); + let mut zf = ZipWriter::new(out_file); + let mut seen = HashSet::new(); + + for entry in &entries { + write_entry(&mut zf, entry, options, &mut seen)?; + } + + let mut inner = zf.finish().context("finishing zip")?; + inner.flush().context("flushing zip")?; + Ok(()) +} + +/// Parses a `0`/`1` flag value, naming `flag` in the error. +fn parse_bool_flag(flag: &str, value: &str) -> Result<bool, lexopt::Error> { + match value { + "0" => Ok(false), + "1" => Ok(true), + _ => Err(format!("invalid {flag} value (expected 0 or 1): {value}").into()), + } +} + +/// Parses command-line arguments (excluding `argv[0]`). +pub fn parse_args<I, T>(args: I) -> Result<Cli, lexopt::Error> +where + I: IntoIterator<Item = T>, + T: Into<OsString>, +{ + use lexopt::prelude::*; + + let mut positional: Vec<PathBuf> = Vec::new(); + let mut compress_level: u32 = 0; + let mut workspace_name = String::new(); + let mut legacy_external_runfiles = false; + let mut runfiles_dir = "runfiles".to_string(); + let mut platform_pathsep = std::path::MAIN_SEPARATOR_STR.to_string(); + + let mut parser = lexopt::Parser::from_args(args); + while let Some(arg) = parser.next()? { + match arg { + Short('h') | Long("help") => return Ok(Cli::Help), + Long("compression") => { + compress_level = parser.value()?.parse()?; + if compress_level > 9 { + return Err(format!( + "invalid --compression value (expected 0-9): {compress_level}" + ) + .into()); + } + } + Long("workspace-name") => { + workspace_name = parser.value()?.string()?; + } + Long("legacy-external-runfiles") => { + legacy_external_runfiles = + parse_bool_flag("--legacy-external-runfiles", &parser.value()?.string()?)?; + } + Long("runfiles-dir") => { + runfiles_dir = parser.value()?.string()?; + } + Long("target-platform-pathsep") => { + platform_pathsep = parser.value()?.string()?; + } + Value(val) => { + positional.push(PathBuf::from(val)); + } + _ => return Err(arg.unexpected()), + } + } + + let [manifest_path, output_zip]: [PathBuf; 2] = + positional.try_into().map_err(|v: Vec<PathBuf>| { + lexopt::Error::from(format!( + "expected 2 positional arguments (<manifest> <output>), got {}", + v.len() + )) + })?; + + Ok(Cli::Run(ZipOptions { + manifest_path, + output_zip, + compress_level, + workspace_name, + legacy_external_runfiles, + runfiles_dir, + platform_pathsep, + })) +} + +/// Parses CLI arguments (excluding `argv[0]` and subcommand name), creates the zip archive, and +/// returns the process exit code. +pub fn run_cli<I, T>(args: I) -> i32 +where + I: IntoIterator<Item = T>, + T: Into<OsString>, +{ + let options = match parse_args(args) { + Ok(Cli::Help) => { + println!("{USAGE}"); + return 0; + } + Ok(Cli::Run(opts)) => opts, + Err(msg) => { + eprintln!("zipper: error: {msg}"); + return 1; + } + }; + if let Err(err) = create_zip(&options) { + eprintln!("zipper: error: {err:#}"); + return 1; + } + 0 +} + +/// Result of parsing the command line. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Cli { + /// `--help` was requested. + Help, + /// Create a zip archive with the given options. + Run(ZipOptions), +} + +/// How a `File` entry's `content_path` should be stored with respect to symlinks. Parsed from +/// the `is_symlink` manifest field. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SymlinkMode { + /// `"0"`: Store as a regular file read from `content_path` (unless `content_path` itself or + /// its sandbox target is a relative source symlink). + Regular, + /// `"1"`: Store as a symlink whose target is read via `read_link(content_path)`. + Symlink, + /// `"-1"`: Unknown at analysis time (Bazel 7 lacks `File.is_symlink`); treated as `Symlink` + /// when `content_path` is a dangling symlink (`fs::metadata` fails) and `Regular` otherwise. + Infer, +} + +/// What a manifest entry stores. See [`parse_entry`] for the manifest line formats. +#[derive(Debug, Clone, PartialEq, Eq)] +enum EntryKind { + /// An empty regular file (`rf-empty`). + Empty, + /// A symlink whose `target` is relative to the parent directory of `zip_path` inside the + /// zip archive (`symlink`). Takes precedence over other kinds at the same `zip_path`. + Symlink { target: String }, + /// Content read from `content_path` on disk (relative to the current working directory or + /// absolute), stored as a file or symlink according to `symlink_mode`. + File { + content_path: PathBuf, + symlink_mode: SymlinkMode, + }, +} + +/// A parsed line from a zipper manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ManifestEntry { + /// Destination path inside the zip archive, relative to the zip root. + zip_path: String, + kind: EntryKind, +} + +/// Configuration options for creating a zip archive. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ZipOptions { + /// Path to the input manifest file, relative to the current working directory or absolute. + pub manifest_path: PathBuf, + /// Path to the output zip archive, relative to the current working directory or absolute. + pub output_zip: PathBuf, + /// ZIP compression level: `0` for uncompressed (`Stored`), or `1..=9` for `Deflated`. + pub compress_level: u32, + /// Main workspace directory name under `runfiles_dir` (e.g. `_main`) prepended to main-repo + /// runfile paths. + pub workspace_name: String, + /// Whether `--legacy_external_runfiles` is enabled, stripping leading `external/` prefixes + /// from external repository runfile short paths. + pub legacy_external_runfiles: bool, + /// Directory prefix within the zip archive for runfiles entries, relative to the zip root + /// (typically `"runfiles"`). + pub runfiles_dir: String, + /// Path separator for the target platform (`"/"` on Unix, `"\\"` on Windows), used to + /// normalize symlink target paths stored in the zip archive. + pub platform_pathsep: String, +}
diff --git a/python/private/zipapp/py_zipapp_rule.bzl b/python/private/zipapp/py_zipapp_rule.bzl index 06ed25a..be00b04 100644 --- a/python/private/zipapp/py_zipapp_rule.bzl +++ b/python/private/zipapp/py_zipapp_rule.bzl
@@ -177,10 +177,12 @@ is_windows = is_windows_platform(ctx) zipper_args.add("\\" if is_windows else "/", format = "--target-platform-pathsep=%s") + zipper = get_shrubbery_tool(ctx, "zipper", ctx.attr._zipper) actions_run( ctx, - executable = ctx.attr._zipper, - arguments = [manifest, zipper_args], + executable = zipper.executable, + toolchain = zipper.toolchain, + arguments = zipper.prefix_args + [manifest, zipper_args], inputs = inputs.build(), outputs = [output], mnemonic = "PyZipAppCreateZip",
diff --git a/tests/exe_zip_maker/exe_zip_maker_test.rs b/tests/exe_zip_maker/exe_zip_maker_test.rs index 76ce96d..1ba26dc 100644 --- a/tests/exe_zip_maker/exe_zip_maker_test.rs +++ b/tests/exe_zip_maker/exe_zip_maker_test.rs
@@ -36,11 +36,11 @@ #[test] fn test_compute_file_sha256_hex() { let temp_dir = env::temp_dir().join(format!("sha256_test_{}", std::process::id())); - fs::create_dir_all(&temp_dir).unwrap(); + fs::create_dir_all(&temp_dir).expect("create temp dir"); let file_path = temp_dir.join("sample.txt"); - fs::write(&file_path, b"hello world\n").unwrap(); - let hash = compute_file_sha256_hex(&file_path).unwrap(); + fs::write(&file_path, b"hello world\n").expect("write file_path"); + let hash = compute_file_sha256_hex(&file_path).expect("hash file"); assert_eq!( hash, "a948904f2f0f479b8f8197694b30184b0d2ed1c1cd2a1ec0fb85d299a192a447" @@ -52,26 +52,26 @@ #[test] fn test_create_exe_zip_successful() { let temp_dir = env::temp_dir().join(format!("create_exe_zip_test_{}", std::process::id())); - fs::create_dir_all(&temp_dir).unwrap(); + fs::create_dir_all(&temp_dir).expect("create temp dir"); let preamble_path = temp_dir.join("preamble.sh"); let zip_path = temp_dir.join("data.zip"); let output_path = temp_dir.join("output.exe"); let zip_content = b"PK\x03\x04dummyzipcontent"; - fs::write(&zip_path, zip_content).unwrap(); + fs::write(&zip_path, zip_content).expect("write zip_path"); let preamble_text = b"#!/bin/bash\nEXPECTED_HASH='%ZIP_HASH%'\n# ... logic ...\n"; - fs::write(&preamble_path, preamble_text).unwrap(); + fs::write(&preamble_path, preamble_text).expect("write preamble_path"); - create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap(); + create_exe_zip(&preamble_path, &zip_path, &output_path).expect("create_exe_zip"); assert!(output_path.exists()); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let st = fs::metadata(&output_path).unwrap(); + let st = fs::metadata(&output_path).expect("stat output"); assert_ne!( st.permissions().mode() & 0o100, 0, @@ -79,7 +79,7 @@ ); } - let content = fs::read(&output_path).unwrap(); + let content = fs::read(&output_path).expect("read output"); let expected_hash = "65e39989ca91c49484998aa3f0429f6943c029609bfd2f3c18c77bf9ded72c59"; let expected_preamble = replace_bytes(preamble_text, PLACEHOLDER, expected_hash.as_bytes()); @@ -93,21 +93,21 @@ #[test] fn test_create_exe_zip_multiple_placeholders() { let temp_dir = env::temp_dir().join(format!("create_exe_zip_multi_{}", std::process::id())); - fs::create_dir_all(&temp_dir).unwrap(); + fs::create_dir_all(&temp_dir).expect("create temp dir"); let preamble_path = temp_dir.join("preamble.sh"); let zip_path = temp_dir.join("data.zip"); let output_path = temp_dir.join("output.exe"); let zip_content = b"PK\x03\x04dummyzipcontent"; - fs::write(&zip_path, zip_content).unwrap(); + fs::write(&zip_path, zip_content).expect("write zip_path"); let preamble_text = b"# First: %ZIP_HASH%\n# Second: %ZIP_HASH%\n"; - fs::write(&preamble_path, preamble_text).unwrap(); + fs::write(&preamble_path, preamble_text).expect("write preamble_path"); - create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap(); + create_exe_zip(&preamble_path, &zip_path, &output_path).expect("create_exe_zip"); - let content = fs::read(&output_path).unwrap(); + let content = fs::read(&output_path).expect("read output"); let expected_hash = "65e39989ca91c49484998aa3f0429f6943c029609bfd2f3c18c77bf9ded72c59"; let expected_preamble = replace_bytes(preamble_text, PLACEHOLDER, expected_hash.as_bytes()); @@ -120,16 +120,16 @@ #[test] fn test_create_exe_zip_creates_parent_dir() { let temp_dir = env::temp_dir().join(format!("create_exe_zip_parent_{}", std::process::id())); - fs::create_dir_all(&temp_dir).unwrap(); + fs::create_dir_all(&temp_dir).expect("create temp dir"); let preamble_path = temp_dir.join("preamble.sh"); let zip_path = temp_dir.join("data.zip"); let output_path = temp_dir.join("nested").join("sub").join("output.exe"); - fs::write(&zip_path, b"content").unwrap(); - fs::write(&preamble_path, b"preamble").unwrap(); + fs::write(&zip_path, b"content").expect("write zip_path"); + fs::write(&preamble_path, b"preamble").expect("write preamble_path"); - create_exe_zip(&preamble_path, &zip_path, &output_path).unwrap(); + create_exe_zip(&preamble_path, &zip_path, &output_path).expect("create_exe_zip"); assert!(output_path.exists()); let _ = fs::remove_dir_all(&temp_dir); @@ -138,17 +138,17 @@ #[test] fn test_create_exe_zip_missing_files() { let temp_dir = env::temp_dir().join(format!("create_exe_zip_err_{}", std::process::id())); - fs::create_dir_all(&temp_dir).unwrap(); + fs::create_dir_all(&temp_dir).expect("create temp dir"); let missing_preamble = temp_dir.join("nonexistent_preamble.sh"); let zip_path = temp_dir.join("data.zip"); let output_path = temp_dir.join("output.exe"); - fs::write(&zip_path, b"dummy").unwrap(); + fs::write(&zip_path, b"dummy").expect("write zip_path"); assert!(create_exe_zip(&missing_preamble, &zip_path, &output_path).is_err()); let preamble_path = temp_dir.join("preamble.sh"); - fs::write(&preamble_path, b"preamble").unwrap(); + fs::write(&preamble_path, b"preamble").expect("write preamble_path"); let missing_zip = temp_dir.join("nonexistent_data.zip"); assert!(create_exe_zip(&preamble_path, &missing_zip, &output_path).is_err()); @@ -172,12 +172,12 @@ assert!(parse_args(["--unknown", "b", "c"]).is_err()); let temp_dir = env::temp_dir().join(format!("exe_zip_cli_{}", std::process::id())); - fs::create_dir_all(&temp_dir).unwrap(); + fs::create_dir_all(&temp_dir).expect("create temp dir"); let preamble_path = temp_dir.join("preamble.sh"); let zip_path = temp_dir.join("data.zip"); let output_path = temp_dir.join("output.exe"); - fs::write(&preamble_path, b"#!/bin/sh\n# %ZIP_HASH%\n").unwrap(); - fs::write(&zip_path, b"PK\x03\x04dummy").unwrap(); + fs::write(&preamble_path, b"#!/bin/sh\n# %ZIP_HASH%\n").expect("write preamble_path"); + fs::write(&zip_path, b"PK\x03\x04dummy").expect("write zip_path"); assert_eq!(run_cli(["missing_args"]), 1); assert_eq!(
diff --git a/tests/integration/prebuilt_toolchain_test.py b/tests/integration/prebuilt_toolchain_test.py index 453ae5b..1c61188 100644 --- a/tests/integration/prebuilt_toolchain_test.py +++ b/tests/integration/prebuilt_toolchain_test.py
@@ -80,16 +80,18 @@ "//:zipapp_test", ) - aquery_res = self.run_bazel( - "aquery", - override_flag, - _NEED_SHRUBBERY_FLAG, - "mnemonic(PyZipAppCreateExecutableZip, //:zipapp_test)", - ) - self.assert_result_matches( - aquery_res, - r"rules_python_shrubbery_prebuilt_file/file/downloaded", - ) + for mnemonic in ("PyZipAppCreateExecutableZip", "PyZipAppCreateZip"): + with self.subTest(mnemonic=mnemonic): + aquery_res = self.run_bazel( + "aquery", + override_flag, + _NEED_SHRUBBERY_FLAG, + f"mnemonic({mnemonic}, //:zipapp_test)", + ) + self.assert_result_matches( + aquery_res, + r"rules_python_shrubbery_prebuilt_file/file/downloaded", + ) if __name__ == "__main__":
diff --git a/tests/modules/rules_rust_stub/rust/defs.bzl b/tests/modules/rules_rust_stub/rust/defs.bzl index b9ca0f3..d2d9008 100644 --- a/tests/modules/rules_rust_stub/rust/defs.bzl +++ b/tests/modules/rules_rust_stub/rust/defs.bzl
@@ -38,3 +38,11 @@ name = name, tags = ["manual"], ) + +def _noop_aspect_impl(_target, _ctx): + return [] + +# The root .bazelrc applies these aspects on every build (see the `rust-lint` +# config). They do nothing here because the stub has no Rust targets to lint. +rust_clippy_aspect = aspect(implementation = _noop_aspect_impl) +rustfmt_aspect = aspect(implementation = _noop_aspect_impl)
diff --git a/tests/modules/rules_rust_stub/rust/settings/BUILD.bazel b/tests/modules/rules_rust_stub/rust/settings/BUILD.bazel new file mode 100644 index 0000000..91ce64c --- /dev/null +++ b/tests/modules/rules_rust_stub/rust/settings/BUILD.bazel
@@ -0,0 +1,10 @@ +load("@bazel_skylib//rules:common_settings.bzl", "string_list_flag") + +package(default_visibility = ["//visibility:public"]) + +# The root .bazelrc sets this flag on every build (see the `rust-lint` config); +# it is unused in WORKSPACE mode because the stub has no Rust targets. +string_list_flag( + name = "clippy_flags", + build_setting_default = [], +)
diff --git a/tests/shrubbery/shrubbery_test.rs b/tests/shrubbery/shrubbery_test.rs index e10de69..7a03035 100644 --- a/tests/shrubbery/shrubbery_test.rs +++ b/tests/shrubbery/shrubbery_test.rs
@@ -11,13 +11,13 @@ .unwrap_or_else(env::temp_dir); let dir = base.join(format!("shrubbery_{}_{}", prefix, std::process::id())); let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); + fs::create_dir_all(&dir).expect("create temp dir"); dir } #[test] fn test_subcommands_registry_sorted_and_complete() { - assert_eq!(SUBCOMMANDS.len(), 1); + assert_eq!(SUBCOMMANDS.len(), 2); let mut sorted = SUBCOMMANDS.to_vec(); sorted.sort_unstable(); assert_eq!(SUBCOMMANDS, sorted.as_slice()); @@ -43,20 +43,20 @@ let zip_in = dir.join("in.zip"); let out_pyz = dir.join("out.pyz"); - fs::write(&preamble, b"#!/bin/sh\n").unwrap(); + fs::write(&preamble, b"#!/bin/sh\n").expect("write preamble"); fs::write( &zip_in, b"PK\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", ) - .unwrap(); + .expect("write zip_in"); assert_eq!( run_cli([ "shrubbery", "exe_zip_maker", - preamble.to_str().unwrap(), - zip_in.to_str().unwrap(), - out_pyz.to_str().unwrap(), + preamble.to_str().expect("preamble path is UTF-8"), + zip_in.to_str().expect("zip_in path is UTF-8"), + out_pyz.to_str().expect("out_pyz path is UTF-8"), ]), 0 ); @@ -65,11 +65,12 @@ #[test] fn test_cli_binary_subprocess() { - let resolved = PathBuf::from( - env::var_os("SHRUBBERY_BIN").expect("SHRUBBERY_BIN must be set"), - ); + let resolved = PathBuf::from(env::var_os("SHRUBBERY_BIN").expect("SHRUBBERY_BIN must be set")); - let help_out = Command::new(&resolved).arg("--help").output().unwrap(); + let help_out = Command::new(&resolved) + .arg("--help") + .output() + .expect("run shrubbery --help"); assert_eq!(help_out.status.code(), Some(0)); let stdout = String::from_utf8_lossy(&help_out.stdout); for sub in SUBCOMMANDS { @@ -79,7 +80,7 @@ let bad_out = Command::new(&resolved) .arg("not_a_subcommand") .output() - .unwrap(); + .expect("run shrubbery with bad subcommand"); assert_eq!(bad_out.status.code(), Some(1)); assert!( String::from_utf8_lossy(&bad_out.stderr).contains("unknown subcommand"),
diff --git a/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl b/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl index 6b336ce..7a6f639 100644 --- a/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl +++ b/tests/shrubbery_toolchain/shrubbery_toolchain_tests.bzl
@@ -28,6 +28,7 @@ _COMPATIBLE_WITH = SUPPORTS_BZLMOD + NOT_WINDOWS _SHRUBBERY_SUFFIX = "crates/shrubbery/shrubbery" _PYTHON_EXE_ZIP_MAKER_SUFFIX = "tools/zipapp/exe_zip_maker_.py" +_PYTHON_ZIPPER_SUFFIX = "tools/zipapp/zipper_.py" _NEED_SHRUBBERY_FLAG = str(Label("//python/private/toolchains:need_shrubbery")) def _define_py_zipapp_subject(name): @@ -67,13 +68,15 @@ ) def _test_py_zipapp_uses_toolchain_shrubbery_impl(env, target): - action = env.expect.that_target(target).action_named( - "PyZipAppCreateExecutableZip", - ) - action.argv().contains_predicate( - matching.str_endswith(_SHRUBBERY_SUFFIX), - ) - action.argv().contains("exe_zip_maker") + for mnemonic, subcommand in [ + ("PyZipAppCreateExecutableZip", "exe_zip_maker"), + ("PyZipAppCreateZip", "zipper"), + ]: + action = env.expect.that_target(target).action_named(mnemonic) + action.argv().contains_predicate( + matching.str_endswith(_SHRUBBERY_SUFFIX), + ) + action.argv().contains(subcommand) _tests.append(_test_py_zipapp_uses_toolchain_shrubbery) @@ -90,12 +93,14 @@ ) def _test_py_zipapp_flag_disabled_uses_python_fallbacks_impl(env, target): - action = env.expect.that_target(target).action_named( - "PyZipAppCreateExecutableZip", - ) - action.argv().contains_predicate( - matching.str_endswith(_PYTHON_EXE_ZIP_MAKER_SUFFIX), - ) + for mnemonic, py_suffix in [ + ("PyZipAppCreateExecutableZip", _PYTHON_EXE_ZIP_MAKER_SUFFIX), + ("PyZipAppCreateZip", _PYTHON_ZIPPER_SUFFIX), + ]: + action = env.expect.that_target(target).action_named(mnemonic) + action.argv().contains_predicate( + matching.str_endswith(py_suffix), + ) _tests.append(_test_py_zipapp_flag_disabled_uses_python_fallbacks)
diff --git a/tests/zipper/BUILD.bazel b/tests/zipper/BUILD.bazel new file mode 100644 index 0000000..8c409e0 --- /dev/null +++ b/tests/zipper/BUILD.bazel
@@ -0,0 +1,15 @@ +load("@rules_rust//rust:defs.bzl", "rust_test") + +package(default_visibility = ["//:__subpackages__"]) + +licenses(["notice"]) + +rust_test( + name = "zipper_test", + size = "small", + srcs = ["zipper_test.rs"], + deps = [ + "//crates/zipper:zipper_lib", + "@crates//:zip", + ], +)
diff --git a/tests/zipper/zipper_test.rs b/tests/zipper/zipper_test.rs new file mode 100644 index 0000000..37fdbce --- /dev/null +++ b/tests/zipper/zipper_test.rs
@@ -0,0 +1,794 @@ +use std::collections::HashMap; +use std::env; +use std::fs::{self, File}; +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use zip::{CompressionMethod, DateTime, ZipArchive}; +use zipper_lib::{create_zip, parse_args, relative_path, Cli, ZipOptions}; + +static TEST_COUNTER: AtomicUsize = AtomicUsize::new(0); + +/// Reads all entries from a ZIP file on disk using `zip::ZipArchive` (which verifies CRC-32). +fn read_zip_entries(zip_path: &Path) -> Vec<ParsedZipEntry> { + let file = File::open(zip_path).expect("open output zip"); + let mut archive = ZipArchive::new(file).expect("parse output zip"); + let mut entries = Vec::with_capacity(archive.len()); + + for i in 0..archive.len() { + let mut entry = archive.by_index(i).expect("read zip entry"); + let mut data = Vec::with_capacity(entry.size() as usize); + entry.read_to_end(&mut data).expect("read zip entry data"); + assert_eq!( + data.len() as u64, + entry.size(), + "uncompressed size mismatch for {}", + entry.name() + ); + + entries.push(ParsedZipEntry { + name: entry.name().to_string(), + compression: entry.compression(), + last_modified: entry.last_modified(), + compress_size: entry.compressed_size(), + file_size: entry.size(), + unix_mode: entry.unix_mode(), + is_symlink: entry.is_symlink(), + data, + }); + } + + entries +} + +fn default_options(manifest_path: &Path, output_zip: &Path) -> ZipOptions { + ZipOptions { + manifest_path: manifest_path.to_path_buf(), + output_zip: output_zip.to_path_buf(), + compress_level: 0, + workspace_name: "my_ws".to_string(), + legacy_external_runfiles: false, + runfiles_dir: "runfiles".to_string(), + platform_pathsep: "/".to_string(), + } +} + +#[test] +fn test_create_zip_with_files_and_symlinks() { + let tmp = TempDir::new("files_and_symlinks"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1_path = tmp.path().join("file1.txt"); + fs::write(&file1_path, b"content1").expect("write file1_path"); + + #[cfg(unix)] + let symlink_path = { + let p = tmp.path().join("symlink_source"); + std::os::unix::fs::symlink("target.txt", &p).expect("create symlink_source"); + p + }; + + let manifest = [ + format!("regular|0|file1.txt|{}", file1_path.display()), + format!("rf-file|0|foo/bar.txt|{}", file1_path.display()), + format!("rf-root-symlink|0|root_file|{}", file1_path.display()), + "rf-empty|empty_file".to_string(), + ] + .join("\n"); + // Rebinding (rather than `Vec::push` under `#[cfg(unix)]`) keeps clippy's + // `useless_vec`/`unused_mut` lints quiet on Windows. + #[cfg(unix)] + let manifest = format!("{manifest}\nrf-symlink|1|link1|{}", symlink_path.display()); + + fs::write(&manifest_path, manifest).expect("write manifest"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + let by_name: HashMap<String, ParsedZipEntry> = + entries.into_iter().map(|e| (e.name.clone(), e)).collect(); + + assert_eq!(by_name["file1.txt"].text(), "content1"); + assert!(!by_name["file1.txt"].is_symlink); + + assert_eq!(by_name["runfiles/my_ws/foo/bar.txt"].text(), "content1"); + assert!(!by_name["runfiles/my_ws/foo/bar.txt"].is_symlink); + + assert_eq!(by_name["runfiles/root_file"].text(), "content1"); + assert!(!by_name["runfiles/root_file"].is_symlink); + + assert_eq!(by_name["runfiles/my_ws/empty_file"].text(), ""); + assert!(!by_name["runfiles/my_ws/empty_file"].is_symlink); + assert_eq!( + by_name["runfiles/my_ws/empty_file"] + .unix_mode + .expect("entry has unix mode") + & 0o777, + 0o644 + ); + + #[cfg(unix)] + { + assert!(by_name["runfiles/my_ws/link1"].is_symlink); + assert_eq!(by_name["runfiles/my_ws/link1"].text(), "target.txt"); + } +} + +#[test] +fn test_create_zip_with_direct_symlink() { + let tmp = TempDir::new("direct_symlink"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + fs::write(&manifest_path, "symlink|path/to/link|target/path\n").expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].name, "runfiles/path/to/link"); + assert!(entries[0].is_symlink); + assert_eq!(entries[0].text(), "../../target/path"); +} + +#[cfg(unix)] +#[test] +fn test_create_zip_with_source_symlink_marked_as_regular() { + let tmp = TempDir::new("source_symlink_regular"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let target_path = tmp.path().join("python3.14"); + fs::write(&target_path, b"python").expect("write target_path"); + let symlink_path = tmp.path().join("python"); + std::os::unix::fs::symlink("python3.14", &symlink_path).expect("create symlink symlink_path"); + + fs::write( + &manifest_path, + format!("rf-file|0|bin/python|{}\n", symlink_path.display()), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].name, "runfiles/my_ws/bin/python"); + assert!(entries[0].is_symlink); + assert_eq!(entries[0].text(), "python3.14"); +} + +#[cfg(unix)] +#[test] +fn test_create_zip_with_sandboxed_source_symlink() { + let tmp = TempDir::new("sandboxed_source_symlink"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let source_dir = tmp.path().join("source"); + fs::create_dir_all(&source_dir).expect("create source_dir"); + let target_path = source_dir.join("python3.14"); + fs::write(&target_path, b"python").expect("write target_path"); + let source_symlink = source_dir.join("python"); + std::os::unix::fs::symlink("python3.14", &source_symlink) + .expect("create symlink source_symlink"); + + let sandbox_dir = tmp.path().join("sandbox"); + fs::create_dir_all(&sandbox_dir).expect("create sandbox_dir"); + let sandbox_symlink = sandbox_dir.join("python"); + std::os::unix::fs::symlink(&source_symlink, &sandbox_symlink) + .expect("create symlink sandbox_symlink"); + let sandbox_regular = sandbox_dir.join("python3.14"); + std::os::unix::fs::symlink(&target_path, &sandbox_regular) + .expect("create symlink sandbox_regular"); + + fs::write( + &manifest_path, + format!( + "rf-file|0|bin/python|{}\nrf-file|0|bin/python3.14|{}\n", + sandbox_symlink.display(), + sandbox_regular.display() + ), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + let by_name: HashMap<String, ParsedZipEntry> = + entries.into_iter().map(|e| (e.name.clone(), e)).collect(); + + assert!(by_name["runfiles/my_ws/bin/python"].is_symlink); + assert_eq!(by_name["runfiles/my_ws/bin/python"].text(), "python3.14"); + + assert!(!by_name["runfiles/my_ws/bin/python3.14"].is_symlink); + assert_eq!(by_name["runfiles/my_ws/bin/python3.14"].text(), "python"); +} + +#[cfg(unix)] +#[test] +fn test_create_zip_with_sandboxed_absolute_source_symlink() { + let tmp = TempDir::new("sandboxed_abs_symlink"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let library_dir = tmp.path().join("external"); + fs::create_dir_all(&library_dir).expect("create library_dir"); + let library = library_dir.join("libfoo.so"); + fs::write(&library, b"library").expect("write library"); + + let solib_dir = tmp.path().join("solib"); + fs::create_dir_all(&solib_dir).expect("create solib_dir"); + let solib_link = solib_dir.join("libfoo.so"); + std::os::unix::fs::symlink(&library, &solib_link).expect("create symlink solib_link"); + + let sandbox_dir = tmp.path().join("sandbox"); + fs::create_dir_all(&sandbox_dir).expect("create sandbox_dir"); + let sandbox_link = sandbox_dir.join("libfoo.so"); + std::os::unix::fs::symlink(&solib_link, &sandbox_link).expect("create symlink sandbox_link"); + + fs::write( + &manifest_path, + format!("rf-file|0|_solib/libfoo.so|{}\n", sandbox_link.display()), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].name, "runfiles/my_ws/_solib/libfoo.so"); + assert!(!entries[0].is_symlink); + assert_eq!(entries[0].text(), "library"); +} + +#[cfg(unix)] +#[test] +fn test_is_symlink_minus_one_inference() { + let tmp = TempDir::new("is_symlink_minus_one"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let broken_link = tmp.path().join("dangling_link"); + std::os::unix::fs::symlink("nonexistent_target", &broken_link) + .expect("create symlink broken_link"); + + let existing_file = tmp.path().join("real.txt"); + fs::write(&existing_file, b"real content").expect("write existing_file"); + + fs::write( + &manifest_path, + format!( + "rf-file|-1|dangling|{}\nrf-file|-1|real|{}\n", + broken_link.display(), + existing_file.display() + ), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + let by_name: HashMap<String, ParsedZipEntry> = + entries.into_iter().map(|e| (e.name.clone(), e)).collect(); + + assert!(by_name["runfiles/my_ws/dangling"].is_symlink); + assert_eq!( + by_name["runfiles/my_ws/dangling"].text(), + "nonexistent_target" + ); + assert!(!by_name["runfiles/my_ws/real"].is_symlink); + assert_eq!(by_name["runfiles/my_ws/real"].text(), "real content"); +} + +#[test] +fn test_pathsep_normalization() { + let tmp = TempDir::new("pathsep_norm"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1_path = tmp.path().join("file1.txt"); + fs::write(&file1_path, b"content1").expect("write file1_path"); + + fs::write( + &manifest_path, + format!( + "regular|0|dir\\file.txt|{}\nsymlink|link/path|target/path\n", + file1_path.display() + ), + ) + .expect("write manifest_path"); + + let mut opts = default_options(&manifest_path, &output_zip); + opts.platform_pathsep = "\\".to_string(); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + let by_name: HashMap<String, ParsedZipEntry> = + entries.into_iter().map(|e| (e.name.clone(), e)).collect(); + + assert!(by_name.contains_key("dir/file.txt")); + assert!(by_name["runfiles/link/path"].is_symlink); + assert_eq!(by_name["runfiles/link/path"].text(), "..\\target\\path"); +} + +#[test] +fn test_symlink_precedence() { + let tmp = TempDir::new("symlink_precedence"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1_path = tmp.path().join("file1.txt"); + fs::write(&file1_path, b"content1").expect("write file1_path"); + + fs::write( + &manifest_path, + format!( + "rf-file|0|path/to/file|{}\nsymlink|my_ws/path/to/file|symlink/target\n", + file1_path.display() + ), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].name, "runfiles/my_ws/path/to/file"); + assert!(entries[0].is_symlink); + assert_eq!(entries[0].text(), "../../../symlink/target"); +} + +#[test] +fn test_timestamps_and_permissions_are_deterministic() { + let tmp = TempDir::new("deterministic_meta"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1_path = tmp.path().join("file1.sh"); + fs::write(&file1_path, b"#!/bin/sh\necho hi\n").expect("write file1_path"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&file1_path) + .expect("stat file1_path") + .permissions(); + perms.set_mode(0o755); + fs::set_permissions(&file1_path, perms).expect("chmod file1_path"); + } + + fs::write( + &manifest_path, + format!("regular|0|file1.sh|{}\n", file1_path.display()), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + assert_eq!(entries.len(), 1); + assert_eq!( + entries[0].last_modified, + Some(DateTime::from_date_and_time(1980, 1, 1, 0, 0, 0).expect("valid timestamp")) + ); + assert!(entries[0].unix_mode.is_some()); + #[cfg(unix)] + assert_eq!( + entries[0].unix_mode.expect("entry has unix mode") & 0o777, + 0o755 + ); +} + +#[test] +fn test_runfiles_mapping_with_cross_repo_paths() { + let tmp = TempDir::new("cross_repo"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1_path = tmp.path().join("file1.txt"); + fs::write(&file1_path, b"content1").expect("write file1_path"); + + fs::write( + &manifest_path, + format!( + "rf-file|0|../other_repo/foo.txt|{}\nrf-empty|../other_repo/empty_file\n", + file1_path.display() + ), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + let by_name: HashMap<String, ParsedZipEntry> = + entries.into_iter().map(|e| (e.name.clone(), e)).collect(); + + assert_eq!(by_name.len(), 2); + assert_eq!(by_name["runfiles/other_repo/foo.txt"].text(), "content1"); + assert_eq!(by_name["runfiles/other_repo/empty_file"].text(), ""); +} + +#[test] +fn test_runfiles_mapping_with_legacy_external_paths() { + let tmp = TempDir::new("legacy_external"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1_path = tmp.path().join("file1.txt"); + fs::write(&file1_path, b"content1").expect("write file1_path"); + + fs::write( + &manifest_path, + format!( + "rf-file|0|external/other_repo/foo.txt|{}\nrf-empty|external/other_repo/empty_file\n", + file1_path.display() + ), + ) + .expect("write manifest_path"); + + let mut opts = default_options(&manifest_path, &output_zip); + opts.legacy_external_runfiles = true; + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + let by_name: HashMap<String, ParsedZipEntry> = + entries.into_iter().map(|e| (e.name.clone(), e)).collect(); + + assert_eq!(by_name.len(), 2); + assert_eq!(by_name["runfiles/other_repo/foo.txt"].text(), "content1"); + assert_eq!(by_name["runfiles/other_repo/empty_file"].text(), ""); +} + +#[test] +fn test_output_deterministic_ordering() { + let tmp = TempDir::new("deterministic_order"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1 = tmp.path().join("file1"); + fs::write(&file1, b"1").expect("write file1"); + let file2 = tmp.path().join("file2"); + fs::write(&file2, b"2").expect("write file2"); + let file3 = tmp.path().join("file3"); + fs::write(&file3, b"3").expect("write file3"); + + fs::write( + &manifest_path, + [ + format!("regular|0|z/regular|{}", file1.display()), + format!("rf-file|0|b_rf_file|{}", file2.display()), + format!("rf-root-symlink|0|a_root_link|{}", file3.display()), + format!("regular|0|a/regular|{}", file3.display()), + "rf-empty|d_rf_empty".to_string(), + format!("rf-symlink|0|c_rf_link|{}", file3.display()), + ] + .join("\n"), + ) + .expect("write manifest"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let names: Vec<String> = read_zip_entries(&output_zip) + .into_iter() + .map(|e| e.name) + .collect(); + + assert_eq!( + names, + vec![ + "a/regular", + "runfiles/a_root_link", + "runfiles/my_ws/b_rf_file", + "runfiles/my_ws/c_rf_link", + "runfiles/my_ws/d_rf_empty", + "z/regular", + ] + ); +} + +#[test] +fn test_deflated_compression() { + let tmp = TempDir::new("deflated"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1 = tmp.path().join("repeated.txt"); + let content = "hello world! ".repeat(500); + fs::write(&file1, content.as_bytes()).expect("write file1"); + + fs::write( + &manifest_path, + format!( + "regular|0|repeated.txt|{}\nrf-empty|empty\nsymlink|a/link|b/target\n", + file1.display() + ), + ) + .expect("write manifest_path"); + + let mut opts = default_options(&manifest_path, &output_zip); + opts.compress_level = 6; + create_zip(&opts).expect("create_zip"); + + let entries = read_zip_entries(&output_zip); + let by_name: HashMap<String, ParsedZipEntry> = + entries.into_iter().map(|e| (e.name.clone(), e)).collect(); + + let rep = &by_name["repeated.txt"]; + assert_eq!(rep.compression, CompressionMethod::Deflated); + assert!(rep.compress_size < rep.file_size); + assert_eq!(rep.text(), content); + + let emp = &by_name["runfiles/my_ws/empty"]; + assert_eq!(emp.compression, CompressionMethod::Deflated); + assert_eq!(emp.text(), ""); + + let sym = &by_name["runfiles/a/link"]; + assert_eq!(sym.compression, CompressionMethod::Stored); + assert!(sym.is_symlink); +} + +#[test] +fn test_relative_path_calculations() { + assert_eq!( + relative_path("runfiles/target/path", "runfiles/path/to"), + "../../target/path" + ); + assert_eq!( + relative_path("runfiles/my_ws/same_dir_target", "runfiles/my_ws"), + "same_dir_target" + ); + assert_eq!(relative_path("runfiles/a", "runfiles/a"), "."); +} + +fn expect_run(cli: Cli) -> ZipOptions { + match cli { + Cli::Run(opts) => opts, + Cli::Help => panic!("expected Cli::Run, got Cli::Help"), + } +} + +#[test] +fn test_parse_args_cli() { + let opts = expect_run( + parse_args([ + "manifest.txt", + "out.zip", + "--compression=4", + "--workspace-name=_main", + "--legacy-external-runfiles=1", + "--runfiles-dir=custom_rf", + "--target-platform-pathsep=\\", + ]) + .expect("all flags parse"), + ); + + assert_eq!(opts.manifest_path, PathBuf::from("manifest.txt")); + assert_eq!(opts.output_zip, PathBuf::from("out.zip")); + assert_eq!(opts.compress_level, 4); + assert_eq!(opts.workspace_name, "_main"); + assert!(opts.legacy_external_runfiles); + assert_eq!(opts.runfiles_dir, "custom_rf"); + assert_eq!(opts.platform_pathsep, "\\"); + + let opts2 = expect_run( + parse_args([ + "--compression", + "0", + "--legacy-external-runfiles", + "0", + "m.txt", + "o.zip", + ]) + .expect("space-separated flags parse"), + ); + assert_eq!(opts2.compress_level, 0); + assert!(!opts2.legacy_external_runfiles); + + assert_eq!(parse_args(["--help"]).expect("--help parses"), Cli::Help); + assert_eq!(parse_args(["a", "b", "-h"]).expect("-h parses"), Cli::Help); + + let err = parse_args(["only_one_arg"]) + .expect_err("one positional must fail") + .to_string(); + assert!(err.contains("expected 2 positional"), "{err}"); + let err = parse_args(["a", "b", "--legacy-external-runfiles=2"]) + .expect_err("bad bool flag must fail") + .to_string(); + assert!(err.contains("expected 0 or 1"), "{err}"); + let err = parse_args(["a", "b", "--compression=10"]) + .expect_err("compression 10 must fail") + .to_string(); + assert!(err.contains("expected 0-9"), "{err}"); + let err = parse_args(["a", "b", "--unknown-flag"]) + .expect_err("unknown flag must fail") + .to_string(); + assert!(err.contains("unknown-flag"), "{err}"); +} + +#[cfg(unix)] +#[test] +fn test_symlink_extraction() { + let tmp = TempDir::new("symlink_extraction"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let target_file = tmp.path().join("target_file.txt"); + fs::write(&target_file, b"target content").expect("write target_file"); + + fs::write( + &manifest_path, + [ + format!("rf-file|0|target/path|{}", target_file.display()), + "symlink|my_ws/path/to/link|my_ws/target/path".to_string(), + format!("rf-file|0|same_dir_target|{}", target_file.display()), + "symlink|my_ws/same_dir_link|my_ws/same_dir_target".to_string(), + ] + .join("\n"), + ) + .expect("write manifest"); + + let opts = default_options(&manifest_path, &output_zip); + create_zip(&opts).expect("create_zip"); + + let extract_dir = tmp.path().join("extract"); + fs::create_dir_all(&extract_dir).expect("create extract_dir"); + + for entry in read_zip_entries(&output_zip) { + let extract_path = extract_dir.join(&entry.name); + if let Some(parent) = extract_path.parent() { + fs::create_dir_all(parent).expect("create parent dir"); + } + if entry.is_symlink { + std::os::unix::fs::symlink(entry.text(), &extract_path) + .expect("create symlink extract_path"); + } else { + fs::write(&extract_path, &entry.data).expect("write extract_path"); + } + } + + let link_path = extract_dir.join("runfiles/my_ws/path/to/link"); + assert!(fs::symlink_metadata(&link_path) + .expect("lstat link_path") + .file_type() + .is_symlink()); + assert_eq!( + fs::read_link(&link_path).expect("readlink link_path"), + PathBuf::from("../../target/path") + ); + assert_eq!( + fs::read_to_string(&link_path).expect("read link_path"), + "target content" + ); + + let link2_path = extract_dir.join("runfiles/my_ws/same_dir_link"); + assert!(fs::symlink_metadata(&link2_path) + .expect("lstat link2_path") + .file_type() + .is_symlink()); + assert_eq!( + fs::read_link(&link2_path).expect("readlink link2_path"), + PathBuf::from("same_dir_target") + ); + assert_eq!( + fs::read_to_string(&link2_path).expect("read link2_path"), + "target content" + ); +} + +#[test] +fn test_invalid_manifest_line_returns_error() { + let tmp = TempDir::new("invalid_manifest"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + fs::write(&manifest_path, "unknown_type|foo|bar\n").expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + let err = format!( + "{:#}", + create_zip(&opts).expect_err("malformed manifest must fail") + ); + assert!(err.contains("manifest line 1: unknown entry type"), "{err}"); +} + +#[test] +fn test_invalid_is_symlink_value_returns_error() { + let tmp = TempDir::new("invalid_is_symlink"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + + let file1_path = tmp.path().join("file1.txt"); + fs::write(&file1_path, b"content1").expect("write file1_path"); + fs::write( + &manifest_path, + format!("rf-file|5|foo|{}\n", file1_path.display()), + ) + .expect("write manifest_path"); + + let opts = default_options(&manifest_path, &output_zip); + let err = format!( + "{:#}", + create_zip(&opts).expect_err("bad is_symlink must fail") + ); + assert!( + err.contains("manifest line 1: invalid is_symlink value"), + "{err}" + ); +} + +#[test] +fn test_compress_level_out_of_range_returns_error() { + let tmp = TempDir::new("compress_level_range"); + let manifest_path = tmp.path().join("manifest.txt"); + let output_zip = tmp.path().join("output.zip"); + fs::write(&manifest_path, "rf-empty|empty\n").expect("write manifest_path"); + + let mut opts = default_options(&manifest_path, &output_zip); + opts.compress_level = 10; + let err = format!( + "{:#}", + create_zip(&opts).expect_err("compress_level 10 must fail") + ); + assert!(err.contains("expected 0-9"), "{err}"); + assert!(!output_zip.exists()); +} + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new(prefix: &str) -> Self { + let id = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); + let path = env::temp_dir().join(format!( + "zipper_test_{}_{}_{}", + prefix, + std::process::id(), + id + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).expect("create temp dir"); + Self { path } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } +} + +#[derive(Debug)] +struct ParsedZipEntry { + /// Entry path inside the zip archive, relative to the zip root. + name: String, + compression: CompressionMethod, + last_modified: Option<DateTime>, + compress_size: u64, + file_size: u64, + unix_mode: Option<u32>, + is_symlink: bool, + data: Vec<u8>, +} + +impl ParsedZipEntry { + fn text(&self) -> &str { + std::str::from_utf8(&self.data).expect("entry data is UTF-8") + } +}