# HG changeset patch # User Tom Prince # Date 1552089949 0 # Sat Mar 09 00:05:49 2019 +0000 # Node ID 1823814b5dff39202aca82672b3df2166e1ffd84 # Parent 09d93ca3bfcc963134f7a32c4ac3c630571790fe Bug 1527858: Vendor redo==2.0.3; r=aki There was an error in logging multiple tries. This updates redo to a version with fixed logging. Differential Revision: https://phabricator.services.mozilla.com/D22806 diff --git a/third_party/python/redo/PKG-INFO b/third_party/python/redo/PKG-INFO --- a/third_party/python/redo/PKG-INFO +++ b/third_party/python/redo/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: redo -Version: 2.0.2 +Version: 2.0.3 Summary: Utilities to retry Python callables. Home-page: https://github.com/bhearsum/redo Author: Ben Hearsum Author-email: ben@hearsum.ca License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/third_party/python/redo/redo.egg-info/PKG-INFO b/third_party/python/redo/redo.egg-info/PKG-INFO --- a/third_party/python/redo/redo.egg-info/PKG-INFO +++ b/third_party/python/redo/redo.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: redo -Version: 2.0.2 +Version: 2.0.3 Summary: Utilities to retry Python callables. Home-page: https://github.com/bhearsum/redo Author: Ben Hearsum Author-email: ben@hearsum.ca License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/third_party/python/redo/redo/__init__.py b/third_party/python/redo/redo/__init__.py --- a/third_party/python/redo/redo/__init__.py +++ b/third_party/python/redo/redo/__init__.py @@ -4,16 +4,17 @@ # You can obtain one at http://mozilla.org/MPL/2.0/. # ***** END LICENSE BLOCK ***** import time from functools import wraps from contextlib import contextmanager import logging import random + log = logging.getLogger(__name__) def retrier(attempts=5, sleeptime=10, max_sleeptime=300, sleepscale=1.5, jitter=1): """ A generator function that sleeps between retries, handles exponential backoff and jitter. The action you are retrying is meant to run after retrier yields. @@ -54,17 +55,19 @@ def retrier(attempts=5, sleeptime=10, ma ... n += 1 ... else: ... print("max tries hit") max tries hit """ jitter = jitter or 0 # py35 barfs on the next line if jitter is None if jitter > sleeptime: # To prevent negative sleep times - raise Exception('jitter ({}) must be less than sleep time ({})'.format(jitter, sleeptime)) + raise Exception( + "jitter ({}) must be less than sleep time ({})".format(jitter, sleeptime) + ) sleeptime_real = sleeptime for _ in range(attempts): log.debug("attempt %i/%i", _ + 1, attempts) yield sleeptime_real if jitter: @@ -76,23 +79,35 @@ def retrier(attempts=5, sleeptime=10, ma sleeptime *= sleepscale if sleeptime_real > max_sleeptime: sleeptime_real = max_sleeptime # Don't need to sleep the last time if _ < attempts - 1: - log.debug("sleeping for %.2fs (attempt %i/%i)", sleeptime_real, _ + 1, attempts) + log.debug( + "sleeping for %.2fs (attempt %i/%i)", sleeptime_real, _ + 1, attempts + ) time.sleep(sleeptime_real) -def retry(action, attempts=5, sleeptime=60, max_sleeptime=5 * 60, - sleepscale=1.5, jitter=1, retry_exceptions=(Exception,), - cleanup=None, args=(), kwargs={}, log_args=True): +def retry( + action, + attempts=5, + sleeptime=60, + max_sleeptime=5 * 60, + sleepscale=1.5, + jitter=1, + retry_exceptions=(Exception,), + cleanup=None, + args=(), + kwargs={}, + log_args=True, +): """ Calls an action function until it succeeds, or we give up. Args: action (callable): the function to retry attempts (int): maximum number of times to try; defaults to 5 sleeptime (float): how many seconds to sleep between tries; defaults to 60s (one minute) @@ -136,37 +151,42 @@ def retry(action, attempts=5, sleeptime= 1 2 3 'success!' """ assert callable(action) assert not cleanup or callable(cleanup) - action_name = getattr(action, '__name__', action) + action_name = getattr(action, "__name__", action) if log_args and (args or kwargs): - log_attempt_args = ("retry: calling %s with args: %s," - " kwargs: %s, attempt #%d", - action_name, args, kwargs) + log_attempt_args = ( + "retry: calling %s with args: %s," " kwargs: %s, attempt #%d", + action_name, + args, + kwargs, + ) else: - log_attempt_args = ("retry: calling %s, attempt #%d", - action_name) + log_attempt_args = ("retry: calling %s, attempt #%d", action_name) if max_sleeptime < sleeptime: - log.debug("max_sleeptime %d less than sleeptime %d", - max_sleeptime, sleeptime) + log.debug("max_sleeptime %d less than sleeptime %d", max_sleeptime, sleeptime) n = 1 - for _ in retrier(attempts=attempts, sleeptime=sleeptime, - max_sleeptime=max_sleeptime, sleepscale=sleepscale, - jitter=jitter): + for _ in retrier( + attempts=attempts, + sleeptime=sleeptime, + max_sleeptime=max_sleeptime, + sleepscale=sleepscale, + jitter=jitter, + ): try: logfn = log.info if n != 1 else log.debug - log_attempt_args += (n, ) - logfn(*log_attempt_args) + logfn_args = log_attempt_args + (n,) + logfn(*logfn_args) return action(*args, **kwargs) except retry_exceptions: log.debug("retry: Caught exception: ", exc_info=True) if cleanup: cleanup() if n == attempts: log.info("retry: Giving up on %s", action_name) raise @@ -198,22 +218,24 @@ def retriable(*retry_args, **retry_kwarg ... raise ValueError("count too small") ... return "success!" >>> foo() 1 2 3 'success!' """ + def _retriable_factory(func): @wraps(func) def _retriable_wrapper(*args, **kwargs): - return retry(func, args=args, kwargs=kwargs, *retry_args, - **retry_kwargs) + return retry(func, args=args, kwargs=kwargs, *retry_args, **retry_kwargs) + return _retriable_wrapper + return _retriable_factory @contextmanager def retrying(func, *retry_args, **retry_kwargs): """ A context manager for wrapping functions with retry functionality. diff --git a/third_party/python/redo/redo/cmd.py b/third_party/python/redo/redo/cmd.py --- a/third_party/python/redo/redo/cmd.py +++ b/third_party/python/redo/redo/cmd.py @@ -12,43 +12,59 @@ from redo import retrying log = logging.getLogger(__name__) def main(argv): from argparse import ArgumentParser, REMAINDER parser = ArgumentParser() parser.add_argument( - "-a", "--attempts", type=int, default=5, - help="How many times to retry.") + "-a", "--attempts", type=int, default=5, help="How many times to retry." + ) + parser.add_argument( + "-s", + "--sleeptime", + type=int, + default=60, + help="How long to sleep between attempts. Sleeptime doubles after each attempt.", + ) parser.add_argument( - "-s", "--sleeptime", type=int, default=60, - help="How long to sleep between attempts. Sleeptime doubles after each attempt.") + "-m", + "--max-sleeptime", + type=int, + default=5 * 60, + help="Maximum length of time to sleep between attempts (limits backoff length).", + ) + parser.add_argument("-v", "--verbose", action="store_true", default=False) parser.add_argument( - "-m", "--max-sleeptime", type=int, default=5*60, - help="Maximum length of time to sleep between attempts (limits backoff length).") - parser.add_argument("-v", "--verbose", action="store_true", default=False) - parser.add_argument("cmd", nargs=REMAINDER, help="Command to run. Eg: wget http://blah") + "cmd", nargs=REMAINDER, help="Command to run. Eg: wget http://blah" + ) args = parser.parse_args(argv[1:]) if args.verbose: logging.basicConfig(level=logging.INFO) logging.getLogger("retry").setLevel(logging.INFO) else: logging.basicConfig(level=logging.ERROR) logging.getLogger("retry").setLevel(logging.ERROR) try: - with retrying(check_call, attempts=args.attempts, sleeptime=args.sleeptime, - max_sleeptime=args.max_sleeptime, - retry_exceptions=(CalledProcessError,)) as r_check_call: + with retrying( + check_call, + attempts=args.attempts, + sleeptime=args.sleeptime, + max_sleeptime=args.max_sleeptime, + retry_exceptions=(CalledProcessError,), + ) as r_check_call: r_check_call(args.cmd) except KeyboardInterrupt: sys.exit(-1) except Exception as e: - log.error("Unable to run command after %d attempts" % args.attempts, exc_info=True) + log.error( + "Unable to run command after %d attempts" % args.attempts, exc_info=True + ) rc = getattr(e, "returncode", -2) sys.exit(rc) if __name__ == "__main__": main(sys.argv) diff --git a/third_party/python/redo/setup.py b/third_party/python/redo/setup.py --- a/third_party/python/redo/setup.py +++ b/third_party/python/redo/setup.py @@ -1,18 +1,16 @@ try: from setuptools import setup except ImportError: from distutils.core import setup setup( name="redo", - version="2.0.2", + version="2.0.3", description="Utilities to retry Python callables.", author="Ben Hearsum", author_email="ben@hearsum.ca", packages=["redo"], - entry_points={ - "console_scripts": ["retry = redo.cmd:main"], - }, + entry_points={"console_scripts": ["retry = redo.cmd:main"]}, url="https://github.com/bhearsum/redo", ) diff --git a/third_party/python/requirements.in b/third_party/python/requirements.in --- a/third_party/python/requirements.in +++ b/third_party/python/requirements.in @@ -3,12 +3,13 @@ blessings==1.7 jsmin==2.1.0 json-e==2.7.0 pathlib2==2.3.2 pip-tools==3.0.0 pipenv==2018.5.18 psutil==5.4.3 pytest==3.6.2 python-hglib==2.4 +redo==2.0.3 requests==2.9.1 six==1.10.0 virtualenv==15.2.0 voluptuous==0.11.5 diff --git a/third_party/python/requirements.txt b/third_party/python/requirements.txt --- a/third_party/python/requirements.txt +++ b/third_party/python/requirements.txt @@ -58,16 +58,19 @@ py==1.5.4 \ --hash=sha256:3fd59af7435864e1a243790d322d763925431213b6b8529c6ca71081ace3bbf7 \ --hash=sha256:e31fb2767eb657cbde86c454f02e99cb846d3cd9d61b318525140214fdc0e98e \ # via pytest pytest==3.6.2 \ --hash=sha256:8ea01fc4fcc8e1b1e305252b4bc80a1528019ab99fd3b88666c9dc38d754406c \ --hash=sha256:90898786b3d0b880b47645bae7b51aa9bbf1e9d1e4510c2cfd15dd65c70ea0cd python-hglib==2.4 \ --hash=sha256:693d6ed92a6566e78802c7a03c256cda33d08c63ad3f00fcfa11379b184b9462 +redo==2.0.3 \ + --hash=sha256:36784bf8ae766e14f9db0e377ccfa02835d648321d2007b6ae0bf4fd612c0f94 \ + --hash=sha256:71161cb0e928d824092a5f16203939bbc0867ce4c4685db263cf22c3ae7634a8 requests==2.9.1 \ --hash=sha256:113fbba5531a9e34945b7d36b33a084e8ba5d0664b703c81a7c572d91919a5b8 \ --hash=sha256:c577815dd00f1394203fc44eb979724b098f88264a9ef898ee45b8e5e9cf587f scandir==1.9.0 \ --hash=sha256:04b8adb105f2ed313a7c2ef0f1cf7aff4871aa7a1883fa4d8c44b5551ab052d6 \ --hash=sha256:1444134990356c81d12f30e4b311379acfbbcd03e0bab591de2696a3b126d58e \ --hash=sha256:1b5c314e39f596875e5a95dd81af03730b338c277c54a454226978d5ba95dbb6 \ --hash=sha256:346619f72eb0ddc4cf355ceffd225fa52506c92a2ff05318cfabd02a144e7c4e \