(out_dir, padded_size, overwrite=False)
Save CIFAR-10 dataset as PNG files
Save CIFAR-10 dataset as PNG files
Save CIFAR - 10 dataset as PNG files
def ingest_cifar10(out_dir, padded_size, overwrite=False):
    """
    Save CIFAR-10 dataset as PNG files
    """
    dataset = dict()
    cifar10 = CIFAR10(path=out_dir, normalize=False)
    dataset['train'], dataset['val'], _ = cifar10.load_data()
    pad_size = (padded_size - 32) // 2 if padded_size > 32 else 0
    pad_width = ((0, 0), (pad_size, pad_size), (pad_size, pad_size))

    set_names = ('train', 'val')
    manifest_files = [os.path.join(out_dir, setn + '-index.csv') for setn in set_names]

    cfg_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'train.cfg')
    log_file = os.path.join(out_dir, 'train.log')
    manifest_list_cfg = ', '.join([k+':'+v for k, v in zip(set_names, manifest_files)])

    with open(cfg_file, 'w') as f:
        f.write('manifest = [{}]\n'.format(manifest_list_cfg))
        f.write('manifest_root = {}\n'.format(out_dir))
        f.write('log = {}\n'.format(log_file))
        f.write('epochs = 165\nrng_seed = 0\nverbose = True\neval_freq = 1\n')
        f.write('backend = gpu\nbatch_size = 64\n')

    if (all([os.path.exists(manifest) for manifest in manifest_files]) and not overwrite):
        print("Found existing manfiest files, skipping ingest, use --overwrite to rerun ingest.")
        return manifest_files

    # Now write out image files and manifests
    for setn, manifest in zip(set_names, manifest_files):
        img_path = os.path.join(out_dir, setn)
        if not os.path.isdir(img_path):
            os.makedirs(img_path)

        records = [('@FILE', 'STRING')]

        for idx, (img, lbl) in enumerate(tqdm(zip(*dataset[setn]))):
            fname = os.path.join(img_path, '{}_{:05d}.png'.format(lbl[0], idx))
            im = np.pad(img.reshape((3, 32, 32)), pad_width, mode='mean')
            im = Image.fromarray(np.uint8(np.transpose(im, axes=[1, 2, 0]).copy()))
            im.save(fname, format='PNG')
            records.append((os.path.relpath(fname, out_dir), lbl[0]))

        np.savetxt(manifest, records, fmt='%s\t%s')

    print("Manifest files written to:\n" + "\n".join(manifest_files))
def ingest_cifar10 ( out_dir , padded_size , overwrite = False ) : dataset = dict ( ) cifar10 = CIFAR10 ( path = out_dir , normalize = False ) dataset [ 'train' ] , dataset [ 'val' ] , _ = cifar10 . load_data ( ) pad_size = ( padded_size - 32 ) // 2 if padded_size > 32 else 0 pad_width = ( ( 0 , 0 ) , ( pad_size , pad_size ) , ( pad_size , pad_size ) ) set_names = ( 'train' , 'val' ) manifest_files = [ os . path . join ( out_dir , setn + '-index.csv' ) for setn in set_names ] cfg_file = os . path . join ( os . path . dirname ( os . path . realpath ( __file__ ) ) , 'train.cfg' ) log_file = os . path . join ( out_dir , 'train.log' ) manifest_list_cfg = ', ' . join ( [ k + ':' + v for k , v in zip ( set_names , manifest_files ) ] ) with open ( cfg_file , 'w' ) as f : f . write ( 'manifest = [{}]\n' . format ( manifest_list_cfg ) ) f . write ( 'manifest_root = {}\n' . format ( out_dir ) ) f . write ( 'log = {}\n' . format ( log_file ) ) f . write ( 'epochs = 165\nrng_seed = 0\nverbose = True\neval_freq = 1\n' ) f . write ( 'backend = gpu\nbatch_size = 64\n' ) if ( all ( [ os . path . exists ( manifest ) for manifest in manifest_files ] ) and not overwrite ) : print ( "Found existing manfiest files, skipping ingest, use --overwrite to rerun ingest." ) return manifest_files # Now write out image files and manifests for setn , manifest in zip ( set_names , manifest_files ) : img_path = os . path . join ( out_dir , setn ) if not os . path . isdir ( img_path ) : os . makedirs ( img_path ) records = [ ( '@FILE' , 'STRING' ) ] for idx , ( img , lbl ) in enumerate ( tqdm ( zip ( * dataset [ setn ] ) ) ) : fname = os . path . join ( img_path , '{}_{:05d}.png' . format ( lbl [ 0 ] , idx ) ) im = np . pad ( img . reshape ( ( 3 , 32 , 32 ) ) , pad_width , mode = 'mean' ) im = Image . fromarray ( np . uint8 ( np . transpose ( im , axes = [ 1 , 2 , 0 ] ) . copy ( ) ) ) im . save ( fname , format = 'PNG' ) records . append ( ( os . path . relpath ( fname , out_dir ) , lbl [ 0 ] ) ) np . savetxt ( manifest , records , fmt = '%s\t%s' ) print ( "Manifest files written to:\n" + "\n" . join ( manifest_files ) )