(dbs, id_range, chunk_size=10000, min_tries=50)
Check each db node for consistent results over a given id range
Check each db node for consistent results over a given id range
Check each db node for consistent results over a given id range
def check_node_integrity(dbs, id_range, chunk_size=10000, min_tries=50):
    """Check each db node for consistent results over a given id range"""
    start_id, end_id = id_range
    next_ids = [start_id] * len(dbs)
    while True:
        for i, db in enumerate(dbs):
            uri = db.uri.rsplit("@")[-1]
            next_id = next_ids[i]
            log.info(f"checking {chunk_size} ids on {uri} starting at {next_id}...")
            reference = None
            for x in range(min_tries):
                ids = query_ids(db, (next_id, end_id), chunk_size)
                if not ids or len(ids) == 1:
                    log.info(f"empty set: {next_id} - {end_id}")
                    assert reference is None, sorted(reference)[:10]
                    return
                if reference is None:
                    reference = ids
                    continue
                if ids != reference:
                    log.warning(f"integrity violation: {uri} on {x + 1} tries")
                    log.debug("diff: %s", reference ^ ids)
                    break
            next_ids[i] = max(reference)
def check_node_integrity ( dbs , id_range , chunk_size = 10000 , min_tries = 50 ) : start_id , end_id = id_range next_ids = [ start_id ] * len ( dbs ) while True : for i , db in enumerate ( dbs ) : uri = db . uri . rsplit ( "@" ) [ - 1 ] next_id = next_ids [ i ] log . info ( f"checking {chunk_size} ids on {uri} starting at {next_id}..." ) reference = None for x in range ( min_tries ) : ids = query_ids ( db , ( next_id , end_id ) , chunk_size ) if not ids or len ( ids ) == 1 : log . info ( f"empty set: {next_id} - {end_id}" ) assert reference is None , sorted ( reference ) [ : 10 ] return if reference is None : reference = ids continue if ids != reference : log . warning ( f"integrity violation: {uri} on {x + 1} tries" ) log . debug ( "diff: %s" , reference ^ ids ) break next_ids [ i ] = max ( reference )