buildifier: improve Windows runner performance (#1404)
* buildifier: improve Windows runner performance
This change aims to improve the `buildidier`'s runner performance on
Windows where it can sometimes take minutes to complete.
It consists in replacing the slow COM `Scripting.FileSystemObject` with
a "native" PowerShell `Get-ChildItem -Recurse` for file discovery, and
batch `buildifier` invocations (100 files at a time) instead of invoking
once per file.
This improves performance on large codebases (like from about 2 minutes
to less than 3 seconds).
Notes:
- cross-process COM calls add latency, whereas `Get-ChildItem` is a
compiled "cmdlet".
- by default, `Get-ChildItem` doesn't recurse into symbolic links to
directories, which is consistent with the current implementation that
explicitly avoids entering `ReparsePoint`s,
- batching files passed to `buildifier` reduces process creation
overhead, with a limit to account for Windows command line length
limitations.
References:
- https://stackoverflow.com/questions/73893997/comments-in-a-long-line-powershell-code-in-a-batch-script
- https://stackoverflow.com/questions/3205027/maximum-length-of-command-line-string
- https://devblogs.microsoft.com/oldnewthing/20031210-00/?p=41553
diff --git a/buildifier/runner.bat.template b/buildifier/runner.bat.template
index 13ca3fa..1d359cd 100644
--- a/buildifier/runner.bat.template
+++ b/buildifier/runner.bat.template
@@ -12,10 +12,8 @@
for /f "tokens=2" %%i in ('findstr /r "\<buildifier\.exe\>" MANIFEST') do (set buildifier_abs_path=%%i)
powershell ^
-function Buildify($Root)^
-{^
- $Folder = (New-Object -Com Scripting.FileSystemObject).GetFolder($Root);^
- $Files = $Folder.Files ^| Where-Object {^
+$Files = Get-ChildItem -LiteralPath '%BUILD_WORKSPACE_DIRECTORY:/=\%' -Recurse -File -ErrorAction SilentlyContinue ^|^
+ Where-Object {^
$_.Name -eq 'BUILD.bazel' `^
-or $_.Name -eq 'BUILD' `^
-or $_.Name -eq 'WORKSPACE' `^
@@ -29,17 +27,11 @@
-or $_.Name -clike 'WORKSPACE.*.bazel' `^
-or $_.Name -clike 'WORKSPACE.*.oss'^
};^
- foreach ($File in $Files)^
- {^
- ^& '%buildifier_abs_path%' %stripped_args% $File.Path;^
- };^
- foreach ($SubFolder in $Folder.Subfolders)^
- {^
- $CurrentItem = Get-Item $SubFolder.Path -ErrorAction SilentlyContinue;^
- if ($CurrentItem -and !$CurrentItem.Attributes.ToString().Contains('ReparsePoint'))^
- {^
- Buildify($SubFolder.Path);^
- };^
- };^
-};^
-Buildify('%BUILD_WORKSPACE_DIRECTORY%');
+ ^<# Process files in batches of 100- to avoid exceeding CreateProcess' maximum length of 32,767 characters #^> ^
+$i = 0;^
+while ($i -lt $Files.Count)^
+{^
+ $Batch = $Files[$i..($i + 99)];^
+ ^& '%buildifier_abs_path%' %stripped_args% $Batch.FullName;^
+ $i += $Batch.Count;^
+};