feat(tar): add ergonomic way to strip_prefix (#829)
diff --git a/docs/tar.md b/docs/tar.md index 5f59d91..cc3a368 100644 --- a/docs/tar.md +++ b/docs/tar.md
@@ -15,13 +15,13 @@ We also provide full control for tar'ring binaries including their runfiles. -## Modifying metadata +## Mutating the tar contents The `mtree_spec` rule can be used to create an mtree manifest for the tar file. -Then you can mutate that spec, as it's just a simple text file, and feed the result +Then you can mutate that spec using `mtree_mutate` and feed the result as the `mtree` attribute of the `tar` rule. -For example, to set the `uid` property, you could: +For example, to set the owner uid of files in the tar, you could: ```starlark mtree_spec( @@ -29,11 +29,10 @@ srcs = ["//some:files"], ) -genrule( +mtree_mutate( name = "change_owner", - srcs = ["mtree"], - outs = ["mtree.mutated"], - cmd = "sed 's/uid=0/uid=1000/' <$< >$@", + mtree = ":mtree", + owner = "1000", ) tar( @@ -43,10 +42,6 @@ ) ``` -Note: We intend to contribute mutation features to https://github.com/vbatts/go-mtree -to provide a richer API for things like `strip_prefix`. -In the meantime, see the `lib/tests/tar/BUILD.bazel` file in this repo for examples. - TODO: - Provide convenience for rules_pkg users to re-use or replace pkg_files trees @@ -95,6 +90,31 @@ | <a id="tar_rule-srcs"></a>srcs | Files, directories, or other targets whose default outputs are placed into the tar.<br><br> If any of the srcs are binaries with runfiles, those are copied into the resulting tar as well. | <a href="https://bazel.build/concepts/labels">List of labels</a> | optional | <code>[]</code> | +<a id="mtree_mutate"></a> + +## mtree_mutate + +<pre> +mtree_mutate(<a href="#mtree_mutate-name">name</a>, <a href="#mtree_mutate-mtree">mtree</a>, <a href="#mtree_mutate-strip_prefix">strip_prefix</a>, <a href="#mtree_mutate-mtime">mtime</a>, <a href="#mtree_mutate-owner">owner</a>, <a href="#mtree_mutate-ownername">ownername</a>, <a href="#mtree_mutate-awk_script">awk_script</a>, <a href="#mtree_mutate-kwargs">kwargs</a>) +</pre> + +Modify metadata in an mtree file. + +**PARAMETERS** + + +| Name | Description | Default Value | +| :------------- | :------------- | :------------- | +| <a id="mtree_mutate-name"></a>name | name of the target, output will be <code>[name].mtree</code>. | none | +| <a id="mtree_mutate-mtree"></a>mtree | input mtree file, typically created by <code>mtree_spec</code>. | none | +| <a id="mtree_mutate-strip_prefix"></a>strip_prefix | prefix to remove from all paths in the tar. Files and directories not under this prefix are dropped. | <code>None</code> | +| <a id="mtree_mutate-mtime"></a>mtime | new modification time for all entries. | <code>None</code> | +| <a id="mtree_mutate-owner"></a>owner | new uid for all entries. | <code>None</code> | +| <a id="mtree_mutate-ownername"></a>ownername | new uname for all entries. | <code>None</code> | +| <a id="mtree_mutate-awk_script"></a>awk_script | may be overridden to change the script containing the modification logic. | <code>"@aspect_bazel_lib//lib/private:modify_mtree.awk"</code> | +| <a id="mtree_mutate-kwargs"></a>kwargs | additional named parameters to genrule | none | + + <a id="tar"></a> ## tar
diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel index a0e0924..1980f3f 100644 --- a/lib/private/BUILD.bazel +++ b/lib/private/BUILD.bazel
@@ -5,6 +5,7 @@ [ "diff_test_tmpl.sh", "diff_test_tmpl.bat", + "modify_mtree.awk", "parse_status_file.jq", "parse_status_file.yq", ],
diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk new file mode 100644 index 0000000..2fe6598 --- /dev/null +++ b/lib/private/modify_mtree.awk
@@ -0,0 +1,32 @@ +# Edits mtree files. See the modify_mtree macro in /lib/tar.bzl. +{ + if (strip_prefix != "") { + if ($1 == strip_prefix) { + # this line declares the directory which is now the root. It may be discarded. + next; + } else if (index($1, strip_prefix) == 1) { + # this line starts with the strip_prefix + sub("^" strip_prefix "/", ""); + } else { + # this line declares some path under a parent directory, which will be discarded + next; + } + } + + if (mtime != "") { + sub(/time=[0-9\.]+/, "time=" mtime); + } + + if (owner != "") { + sub(/uid=[0-9\.]+/, "uid=" owner) + } + + if (ownername != "") { + sub(/uname=[^ ]+/, "uname=" ownername) + } + + if (package_dir != "") { + sub(/^/, package_dir "/") + } + print; +}
diff --git a/lib/tar.bzl b/lib/tar.bzl index e8051a7..30ad7ae 100644 --- a/lib/tar.bzl +++ b/lib/tar.bzl
@@ -13,13 +13,13 @@ We also provide full control for tar'ring binaries including their runfiles. -## Modifying metadata +## Mutating the tar contents The `mtree_spec` rule can be used to create an mtree manifest for the tar file. -Then you can mutate that spec, as it's just a simple text file, and feed the result +Then you can mutate that spec using `mtree_mutate` and feed the result as the `mtree` attribute of the `tar` rule. -For example, to set the `uid` property, you could: +For example, to set the owner uid of files in the tar, you could: ```starlark mtree_spec( @@ -27,11 +27,10 @@ srcs = ["//some:files"], ) -genrule( +mtree_mutate( name = "change_owner", - srcs = ["mtree"], - outs = ["mtree.mutated"], - cmd = "sed 's/uid=0/uid=1000/' <$< >$@", + mtree = ":mtree", + owner = "1000", ) tar( @@ -41,10 +40,6 @@ ) ``` -Note: We intend to contribute mutation features to https://github.com/vbatts/go-mtree -to provide a richer API for things like `strip_prefix`. -In the meantime, see the `lib/tests/tar/BUILD.bazel` file in this repo for examples. - TODO: - Provide convenience for rules_pkg users to re-use or replace pkg_files trees """ @@ -130,3 +125,43 @@ mtree = mtree_target, **kwargs ) + +def mtree_mutate( + name, + mtree, + strip_prefix = None, + mtime = None, + owner = None, + ownername = None, + awk_script = "@aspect_bazel_lib//lib/private:modify_mtree.awk", + **kwargs): + """Modify metadata in an mtree file. + + Args: + name: name of the target, output will be `[name].mtree`. + mtree: input mtree file, typically created by `mtree_spec`. + strip_prefix: prefix to remove from all paths in the tar. Files and directories not under this prefix are dropped. + mtime: new modification time for all entries. + owner: new uid for all entries. + ownername: new uname for all entries. + awk_script: may be overridden to change the script containing the modification logic. + **kwargs: additional named parameters to genrule + """ + vars = [] + if strip_prefix: + vars.append("-v strip_prefix='{}'".format(strip_prefix)) + if mtime: + vars.append("-v mtime='{}'".format(mtime)) + if owner: + vars.append("-v owner='{}'".format(owner)) + if ownername: + vars.append("-v ownername='{}'".format(ownername)) + + native.genrule( + name = name, + srcs = [mtree], + outs = [name + ".mtree"], + cmd = "awk {} -f $(execpath {}) <$< >$@".format(" ".join(vars), awk_script), + tools = [awk_script], + **kwargs + )
diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel index a6c2bb5..497f707 100644 --- a/lib/tests/tar/BUILD.bazel +++ b/lib/tests/tar/BUILD.bazel
@@ -1,6 +1,6 @@ load("@aspect_bazel_lib//lib:copy_directory.bzl", "copy_directory") load("@aspect_bazel_lib//lib:diff_test.bzl", "diff_test") -load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_mutate", "mtree_spec", "tar") load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:write_file.bzl", "write_file") @@ -123,25 +123,10 @@ srcs = _SRCS5, ) -# This is a low-tech way to mutate the mtree specification, just using regex. -# See docs on tar about future directions for mtree mutation -genrule( +mtree_mutate( name = "strip_prefix", - srcs = ["mtree5"], - outs = ["mtree5.stripped"], - # Modify lines starting with the package name, e.g. - # lib/tests/tar/a uid=0 gid=0 time=1672560000 mode=0755 type=file content=bazel-out/darwin_arm64-opt/bin/lib/tests/tar/a - # -> - # a uid=0 gid=0 time=1672560000 mode=0755 type=file content=bazel-out/darwin_arm64-opt/bin/lib/tests/tar/a - cmd = "sed '{}' <$< | sed '/^\\ /d' > $@".format( - "; ".join(reversed([ - "s#^{s}/##; s#^{s}##".format(s = "/".join(package_name().split("/")[:i])) - for (i, _) in enumerate( - package_name().split("/"), - 1, - ) - ])), - ), + mtree = "mtree5", + strip_prefix = package_name(), ) tar( @@ -342,3 +327,30 @@ "drwxrwxrwt 0 0 0 0 Aug 3 2017 ./tmp/", ], ) + +# Case 12: arbitrary mtree modifications +mtree_mutate( + name = "modified1", + mtree = "source-casync.mtree", + strip_prefix = "xattr", +) + +diff_test( + name = "test1", + file1 = "modified1.mtree", + file2 = "expected1.mtree", +) + +mtree_mutate( + name = "modified2", + mtime = 946684740, # 1999-12-31, 23:59 + mtree = "source-casync.mtree", + owner = "123", + ownername = "fred", +) + +diff_test( + name = "test2", + file1 = "modified2.mtree", + file2 = "expected2.mtree", +)
diff --git a/lib/tests/tar/expected1.mtree b/lib/tests/tar/expected1.mtree new file mode 100644 index 0000000..2ee0ab4 --- /dev/null +++ b/lib/tests/tar/expected1.mtree
@@ -0,0 +1,4 @@ +xattr.go type=file mode=0664 size=984 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=a2700b603df30c3b0a91bdcf9045e64aea42f62647b0128ea154f51a0c48991e +xattr_test.go type=file mode=0664 size=859 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=91294ea554801b75f6a9e33c268807c9620b531eb813ea24512dd4eeaf0592e4 +xattr_unsupported.go type=file mode=0664 size=509 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=81ced06a1cdf88c4936d10bbf8d46edc2d42dc26efeed3691ddbeeb469865f8a +xattr_unsupported_test.go type=file mode=0664 size=877 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=46605a03a985c7a3a4ab1a488f81382db4865f77b90b6a2b32693af39a8e1fba
diff --git a/lib/tests/tar/expected2.mtree b/lib/tests/tar/expected2.mtree new file mode 100644 index 0000000..eb253b1 --- /dev/null +++ b/lib/tests/tar/expected2.mtree
@@ -0,0 +1,5 @@ +xattr type=dir mode=0775 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 +xattr/xattr.go type=file mode=0664 size=984 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=a2700b603df30c3b0a91bdcf9045e64aea42f62647b0128ea154f51a0c48991e +xattr/xattr_test.go type=file mode=0664 size=859 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=91294ea554801b75f6a9e33c268807c9620b531eb813ea24512dd4eeaf0592e4 +xattr/xattr_unsupported.go type=file mode=0664 size=509 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=81ced06a1cdf88c4936d10bbf8d46edc2d42dc26efeed3691ddbeeb469865f8a +xattr/xattr_unsupported_test.go type=file mode=0664 size=877 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=46605a03a985c7a3a4ab1a488f81382db4865f77b90b6a2b32693af39a8e1fba
diff --git a/lib/tests/tar/source-casync.mtree b/lib/tests/tar/source-casync.mtree new file mode 100644 index 0000000..5978130 --- /dev/null +++ b/lib/tests/tar/source-casync.mtree
@@ -0,0 +1,5 @@ +xattr type=dir mode=0775 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 +xattr/xattr.go type=file mode=0664 size=984 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=a2700b603df30c3b0a91bdcf9045e64aea42f62647b0128ea154f51a0c48991e +xattr/xattr_test.go type=file mode=0664 size=859 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=91294ea554801b75f6a9e33c268807c9620b531eb813ea24512dd4eeaf0592e4 +xattr/xattr_unsupported.go type=file mode=0664 size=509 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=81ced06a1cdf88c4936d10bbf8d46edc2d42dc26efeed3691ddbeeb469865f8a +xattr/xattr_unsupported_test.go type=file mode=0664 size=877 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=46605a03a985c7a3a4ab1a488f81382db4865f77b90b6a2b32693af39a8e1fba