(data, sizes, samples_by_type, write, summarize=False)
report blob type, number of blobs, total size grouped by domain
report blob type, number of blobs, total size grouped by domain
report blob type number of blobs total size grouped by domain
def report_blob_sizes(data, sizes, samples_by_type, write, summarize=False):
    """report blob type, number of blobs, total size grouped by domain"""
    def iter_headers(by_domain):
        for domain in by_domain:
            yield domain
            yield "MEAN"
            yield "COUNT"

    def iter_sizes(doc_type, domain_sizes, totals):
        for domain in by_domain:
            blob_sizes = domain_sizes[domain]
            numerics = [s.length for s in blob_sizes if s.length is not UNKNOWN]
            if numerics:
                bucket = blob_sizes[0].bucket
                bucket_samples = sum(samples_by_type[bucket].values())
                mean_size = int(mean(numerics))
                est_size = (
                    mean_size *
                    len(data[bucket]) *                 # number blobs in bucket
                    (len(numerics) / bucket_samples)    # porportion of samples
                )
            else:
                mean_size = 0
                est_size = 0
            found_of_total = "{}/{}".format(len(numerics), len(blob_sizes))
            totals[domain]["size"] += est_size
            totals[domain]["found"] += len(numerics)
            totals[domain]["count"] += len(blob_sizes)
            yield sizeof_fmt(est_size)
            yield sizeof_fmt(mean_size)
            yield found_of_total if blob_sizes else "-"  # FOUND/TOTAL

    def iter_totals(totals):
        yield sizeof_fmt(sum(t["size"] for t in totals.values()))
        for domain in by_domain:
            yield sizeof_fmt(totals[domain]["size"])
            yield ""
            yield "{found}/{count}".format(**totals[domain])

    def sumlens(item):
        return -sum(s.length for s in item[1] if s.length is not UNKNOWN)

    if summarize:
        sizes = {"EST SIZE": list(chain.from_iterable(sizes.values()))}

    # get top five domains + all others combined
    OTHER = "OTHER"
    by_domain = OrderedDict()
    by_type = defaultdict(lambda: defaultdict(list))
    for i, (domain, domain_sizes) in enumerate(sorted(sizes.items(), key=sumlens)):
        if i < 5:
            by_domain[domain] = domain_sizes
        else:
            if i == 5:
                by_domain[OTHER] = []
            by_domain[OTHER].extend(domain_sizes)
            domain = OTHER
        for size in domain_sizes:
            by_type[size.doc_type][domain].append(size)

    def key(item):
        return sum(sumlens(["ignored", sizes]) for sizes in item[1].values())

    totals = {domain: {
        "size": 0,
        "found": 0,
        "count": 0,
    } for domain in by_domain}
    if summarize:
        write(["SUMMARY"])
    else:
        write(["Storage use based on sampled estimates (may be inaccurate)"])
    write(["DOC_TYPE"] + list(iter_headers(by_domain)))
    for doc_type, domain_sizes in sorted(by_type.items(), key=key):
        write([doc_type] + list(iter_sizes(doc_type, domain_sizes, totals)))
    write(["---"] + ["---" for x in iter_headers(by_domain)])
    write(list(iter_totals(totals)))
    write([])
def report_blob_sizes ( data , sizes , samples_by_type , write , summarize = False ) : def iter_headers ( by_domain ) : for domain in by_domain : yield domain yield "MEAN" yield "COUNT" def iter_sizes ( doc_type , domain_sizes , totals ) : for domain in by_domain : blob_sizes = domain_sizes [ domain ] numerics = [ s . length for s in blob_sizes if s . length is not UNKNOWN ] if numerics : bucket = blob_sizes [ 0 ] . bucket bucket_samples = sum ( samples_by_type [ bucket ] . values ( ) ) mean_size = int ( mean ( numerics ) ) est_size = ( mean_size * len ( data [ bucket ] ) * # number blobs in bucket ( len ( numerics ) / bucket_samples ) # porportion of samples ) else : mean_size = 0 est_size = 0 found_of_total = "{}/{}" . format ( len ( numerics ) , len ( blob_sizes ) ) totals [ domain ] [ "size" ] += est_size totals [ domain ] [ "found" ] += len ( numerics ) totals [ domain ] [ "count" ] += len ( blob_sizes ) yield sizeof_fmt ( est_size ) yield sizeof_fmt ( mean_size ) yield found_of_total if blob_sizes else "-" # FOUND/TOTAL def iter_totals ( totals ) : yield sizeof_fmt ( sum ( t [ "size" ] for t in totals . values ( ) ) ) for domain in by_domain : yield sizeof_fmt ( totals [ domain ] [ "size" ] ) yield "" yield "{found}/{count}" . format ( * * totals [ domain ] ) def sumlens ( item ) : return - sum ( s . length for s in item [ 1 ] if s . length is not UNKNOWN ) if summarize : sizes = { "EST SIZE" : list ( chain . from_iterable ( sizes . values ( ) ) ) } # get top five domains + all others combined OTHER = "OTHER" by_domain = OrderedDict ( ) by_type = defaultdict ( lambda : defaultdict ( list ) ) for i , ( domain , domain_sizes ) in enumerate ( sorted ( sizes . items ( ) , key = sumlens ) ) : if i < 5 : by_domain [ domain ] = domain_sizes else : if i == 5 : by_domain [ OTHER ] = [ ] by_domain [ OTHER ] . extend ( domain_sizes ) domain = OTHER for size in domain_sizes : by_type [ size . doc_type ] [ domain ] . append ( size ) def key ( item ) : return sum ( sumlens ( [ "ignored" , sizes ] ) for sizes in item [ 1 ] . values ( ) ) totals = { domain : { "size" : 0 , "found" : 0 , "count" : 0 , } for domain in by_domain } if summarize : write ( [ "SUMMARY" ] ) else : write ( [ "Storage use based on sampled estimates (may be inaccurate)" ] ) write ( [ "DOC_TYPE" ] + list ( iter_headers ( by_domain ) ) ) for doc_type , domain_sizes in sorted ( by_type . items ( ) , key = key ) : write ( [ doc_type ] + list ( iter_sizes ( doc_type , domain_sizes , totals ) ) ) write ( [ "---" ] + [ "---" for x in iter_headers ( by_domain ) ] ) write ( list ( iter_totals ( totals ) ) ) write ( [ ] )