Changeset c987be5 in rattail


Ignore:
Timestamp:
07/19/2022 10:39:41 AM (7 months ago)
Author:
Lance Edgar <lance@…>
Branches:
master
Children:
27d18c42
Parents:
9e8fe47
Message:

Add retry loop for starting transaction in datasync consumer

File:
1 edited

Legend:

Unmodified
Added
Removed
  • rattail/datasync/daemon.py

    r9e8fe47 rc987be5  
    570570    log.debug("will process %s changes from %s", len(changes),
    571571              app.localtime(obtained, from_utc=True))
    572     consumer.begin_transaction()
    573 
     572
     573    # first retry loop is to begin the transaction
     574    attempts = 0
     575    errtype = None
     576    while True:
     577        attempts += 1
     578
     579        try:
     580            consumer.begin_transaction()
     581
     582        except Exception as errobj: # processing failed!
     583            exc_type, exc, traceback = sys.exc_info()
     584
     585            # if we've reached our final attempt, stop retrying
     586            if attempts >= consumer.retry_attempts:
     587                log.warning("attempt #%s failed calling `consumer.begin_transaction()`; "
     588                            "this thread will now *terminate* until datasync restart",
     589                            attempts, exc_info=True)
     590                app.send_email('datasync_error_consumer_process_changes', {
     591                    'watcher': consumer.watcher,
     592                    'consumer': consumer,
     593                    'error': exc,
     594                    'attempts': attempts,
     595                    'traceback': ''.join(format_exception(exc_type, exc, traceback)).strip(),
     596                    'datasync_url': config.datasync_url(),
     597                })
     598                return False
     599
     600            # if this exception is not the first, and is of a different type
     601            # than seen previously, do *not* continue to retry
     602            if errtype is not None and not isinstance(errobj, errtype):
     603                log.exception("new exception differs from previous one(s), "
     604                              "giving up on consumer.begin_transaction()")
     605                return False
     606
     607            # record the type of exception seen; maybe pause before next retry
     608            errtype = type(errobj)
     609            log.warning("attempt #%s failed for '%s' -> '%s' consumer.begin_transaction()",
     610                        attempts, consumer.watcher.key, consumer.key)
     611            log.debug("pausing for %s seconds before making attempt #%s of %s",
     612                      consumer.retry_delay, attempts + 1, consumer.retry_attempts)
     613            if consumer.retry_delay:
     614                time.sleep(consumer.retry_delay)
     615
     616        else: # transaction began okay
     617
     618            # can stop the attempt/retry loop now
     619            break
     620
     621    # second retry loop is to process the changes
    574622    attempts = 0
    575623    errtype = None
Note: See TracChangeset for help on using the changeset viewer.