Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 1 | # Copyright 2022 The Pigweed Authors |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| 4 | # use this file except in compliance with the License. You may obtain a copy of |
| 5 | # the License at |
| 6 | # |
| 7 | # https://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations under |
| 13 | # the License. |
| 14 | """Ensure the given call completes before swarming kills the build.""" |
| 15 | |
Rob Mohr | 758a472 | 2024-05-08 22:03:16 +0000 | [diff] [blame] | 16 | from __future__ import annotations |
| 17 | |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 18 | import contextlib |
| 19 | import dataclasses |
Rob Mohr | 8653154 | 2024-08-13 19:33:21 +0000 | [diff] [blame] | 20 | from typing import TYPE_CHECKING |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 21 | |
| 22 | from recipe_engine import recipe_api |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 23 | from RECIPE_MODULES.fuchsia.utils import nice_duration |
| 24 | |
Rob Mohr | 8653154 | 2024-08-13 19:33:21 +0000 | [diff] [blame] | 25 | if TYPE_CHECKING: # pragma: no cover |
| 26 | from typing import Generator |
| 27 | |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 28 | |
| 29 | class TimeoutApi(recipe_api.RecipeApi): |
| 30 | """Ensure the given call completes before swarming kills the build.""" |
| 31 | |
| 32 | @contextlib.contextmanager |
Rob Mohr | 0b9859a | 2024-03-06 23:25:38 +0000 | [diff] [blame] | 33 | def __call__(self, buffer_sec: float = 120) -> Generator[None, None, None]: |
Rob Mohr | 7ff94c0 | 2023-11-20 17:01:28 +0000 | [diff] [blame] | 34 | current_time_sec: float = self.m.time.time() |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 35 | |
Rob Mohr | 52bd039 | 2024-03-25 20:43:10 +0000 | [diff] [blame] | 36 | # Amount of time elapsed in the build. |
Rob Mohr | 7ff94c0 | 2023-11-20 17:01:28 +0000 | [diff] [blame] | 37 | elapsed_time_sec: float = ( |
Rob Mohr | d607119 | 2023-06-15 22:17:01 +0000 | [diff] [blame] | 38 | current_time_sec - self.m.buildbucket.build.start_time.seconds |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 39 | ) |
| 40 | |
Rob Mohr | 52bd039 | 2024-03-25 20:43:10 +0000 | [diff] [blame] | 41 | # Amount of time before the build times out. |
Rob Mohr | 7ff94c0 | 2023-11-20 17:01:28 +0000 | [diff] [blame] | 42 | time_remaining_sec: float = ( |
Rob Mohr | d607119 | 2023-06-15 22:17:01 +0000 | [diff] [blame] | 43 | self.m.buildbucket.build.execution_timeout.seconds |
| 44 | - elapsed_time_sec |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 45 | ) |
| 46 | |
| 47 | # Give a buffer before build times out and kill this step then. This |
| 48 | # should give enough time to read any logfiles and maybe upload to |
Rob Mohr | d607119 | 2023-06-15 22:17:01 +0000 | [diff] [blame] | 49 | # GCS before the build times out. |
Rob Mohr | 52bd039 | 2024-03-25 20:43:10 +0000 | [diff] [blame] | 50 | timeout_sec: float = max(time_remaining_sec - buffer_sec, 10.0) |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 51 | |
Rob Mohr | 52bd039 | 2024-03-25 20:43:10 +0000 | [diff] [blame] | 52 | # Timing often looks weird in recipe unit tests. If the amount of time |
| 53 | # remaining before the build times out is a negative number, ignore it |
| 54 | # and proceed as if everything is internally consistent. |
| 55 | if self._test_data.enabled and time_remaining_sec < 0: |
| 56 | time_remaining_sec = timeout_sec = 10.0 |
| 57 | |
| 58 | # Don't start steps if there's less than 10s remaining. A step timing |
| 59 | # out looks like an infra failure which we don't want. |
| 60 | if time_remaining_sec < 10.0: |
| 61 | self.m.step.empty( |
| 62 | 'timed out', |
| 63 | status=self.m.step.FAILURE, |
| 64 | step_text=( |
| 65 | 'The build has too little time remaining, not starting new ' |
| 66 | f'steps. {time_remaining_sec}s remaining.' |
| 67 | ), |
| 68 | ) |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 69 | |
Rob Mohr | d607119 | 2023-06-15 22:17:01 +0000 | [diff] [blame] | 70 | with self.m.step.nest(f'timeout {nice_duration(timeout_sec)}'): |
Rob Mohr | adee1d4 | 2023-03-28 21:30:26 +0000 | [diff] [blame] | 71 | pass |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 72 | |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 73 | try: |
Rob Mohr | 686cf71 | 2023-06-15 22:19:09 +0000 | [diff] [blame] | 74 | with self.m.time.timeout(timeout_sec): |
Rob Mohr | 8d13e25 | 2022-09-27 20:01:44 +0000 | [diff] [blame] | 75 | yield |
| 76 | |
| 77 | finally: |
| 78 | pass |