Run zap in parallel by default for zap_regen_all.py (#23766)
* Run zap in parallel on regen all. This makes my local zap run in 88 seconds instead of about 222
* Restyle
* Fix arguments for generate.py
diff --git a/scripts/tools/zap/generate.py b/scripts/tools/zap/generate.py
index 5c48a41..9a10752 100755
--- a/scripts/tools/zap/generate.py
+++ b/scripts/tools/zap/generate.py
@@ -32,6 +32,7 @@
templateFile: str
outputDir: str
runBootstrap: bool
+ parallel: bool = True
CHIP_ROOT_DIR = os.path.realpath(
@@ -103,6 +104,9 @@
help='Output directory for the generated files (default: automatically selected)')
parser.add_argument('--run-bootstrap', default=None, action='store_true',
help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--no-parallel', action='store_false', dest='parallel')
+ parser.set_defaults(parallel=True)
args = parser.parse_args()
# By default, this script assumes that the global CHIP template is used with
@@ -259,6 +263,11 @@
# The maximum memory usage is over 4GB (#15620)
os.environ["NODE_OPTIONS"] = "--max-old-space-size=8192"
+
+ if cmdLineArgs.parallel:
+ # Parallel-compatible runs will need separate state
+ os.environ["ZAP_TEMPSTATE"] = "1"
+
runGeneration(cmdLineArgs.zapFile, cmdLineArgs.zclFile, cmdLineArgs.templateFile, cmdLineArgs.outputDir)
prettifiers = [
diff --git a/scripts/tools/zap_regen_all.py b/scripts/tools/zap_regen_all.py
index 18de729..e948308 100755
--- a/scripts/tools/zap_regen_all.py
+++ b/scripts/tools/zap_regen_all.py
@@ -21,6 +21,8 @@
import sys
import subprocess
import logging
+import multiprocessing
+
from dataclasses import dataclass
CHIP_ROOT_DIR = os.path.realpath(
@@ -113,6 +115,11 @@
help="Don't do any generation, just log what targets would be generated (default: False)")
parser.add_argument('--run-bootstrap', default=None, action='store_true',
help='Automatically run ZAP bootstrap. By default the bootstrap is not triggered')
+
+ parser.add_argument('--parallel', action='store_true')
+ parser.add_argument('--no-parallel', action='store_false', dest='parallel')
+ parser.set_defaults(parallel=True)
+
return parser.parse_args()
@@ -253,6 +260,14 @@
return targets
+def _ParallelGenerateOne(target):
+ """
+ Helper method to be passed to multiprocessing parallel generation of
+ items.
+ """
+ target.generate()
+
+
def main():
logging.basicConfig(
level=logging.INFO,
@@ -269,8 +284,15 @@
if args.run_bootstrap:
subprocess.check_call(os.path.join(CHIP_ROOT_DIR, "scripts/tools/zap/zap_bootstrap.sh"), shell=True)
- for target in targets:
- target.generate()
+ if args.parallel:
+ # Ensure each zap run is independent
+ os.environ['ZAP_TEMPSTATE'] = '1'
+ with multiprocessing.Pool() as pool:
+ for _ in pool.imap_unordered(_ParallelGenerateOne, targets):
+ pass
+ else:
+ for target in targets:
+ target.generate()
if __name__ == '__main__':
