(input_dir, out_dir, overwrite=False, train_frac=0.8)
return [manifests[sn] for sn in set_names]
save_ingested_whale_files
save_ingested_whale_files
save_ingested_whale_files
def ingest_whales(input_dir, out_dir, overwrite=False, train_frac=0.8):
    '''
    save_ingested_whale_files
    '''
    zfilename = os.path.join(input_dir, 'whale_data.zip')
    orig_out_dir = out_dir
    out_dir = os.path.join(orig_out_dir, 'whale-extracted')
    set_names = ['all', 'val', 'train', 'noise', 'test']
    manifests = {s: os.path.join(out_dir, s + '-index.csv') for s in set_names}

    manifest_list_cfg = ', '.join([k+':'+v for k, v in manifests.items()])

    cfg_path = os.path.dirname(os.path.realpath(__file__))
    for runtype in ('eval', 'subm'):
        cfg_file = os.path.join(cfg_path, 'whale_' + runtype + '.cfg')
        log_file = os.path.join(orig_out_dir, 'train_' + runtype + '.log')
        with open(cfg_file, 'w') as f:
            f.write('manifest = [{}]\n'.format(manifest_list_cfg))
            f.write('manifest_root = {}\n'.format(out_dir))
            f.write('log = {}\n'.format(log_file))
            f.write('epochs = 4\nrng_seed = 0\nverbose = True\n')
            if runtype == 'subm':
                f.write('save_path = {}\n'.format(os.path.join(orig_out_dir, 'model.p')))
                f.write('submission_file = {}\n'.format(os.path.join(orig_out_dir, 'subm.txt')))

    if (all([os.path.exists(manifests[sn]) for sn in set_names]) and not overwrite):
        return [manifests[sn] for sn in set_names]

    with ZipFile(zfilename, 'r') as zf:
        zf.extractall(out_dir)

        input_csv = os.path.join(out_dir, 'data', 'train.csv')
        train_records = np.genfromtxt(input_csv, delimiter=',', skip_header=1, dtype=None)
        np.random.seed(0)
        np.random.shuffle(train_records)

        pos_list, neg_list = [], []

        for aiff, lbl in train_records:
            try:
                record = (os.path.join('data', 'train', aiff), lbl)
            except Exception:
                record = (os.path.join('data', 'train', aiff.decode()), lbl)
            if lbl == 1:
                pos_list.append(record)
            else:
                neg_list.append(record)

        neg_part, pos_part = int(len(neg_list) * train_frac), int(len(pos_list) * train_frac)

        set_lists = dict()
        set_lists['all'] = neg_list + pos_list
        set_lists['train'] = neg_list[:neg_part] + pos_list[:pos_part]
        set_lists['val'] = neg_list[neg_part:] + pos_list[pos_part:]

        # Use just the files from the non-whale calls to use as a set of noise samples
        set_lists['noise'] = [(a) for a, l in neg_list[:neg_part]]

        # Write out the test files
        set_lists['test'] = [os.path.relpath(fn, out_dir) for fn in glob(
                                                os.path.join(out_dir, 'data', 'test', '*.aiff'))]

        for sn in set_names:
            if sn != 'test':
                np.random.shuffle(set_lists[sn])

            format_str = '%s' if sn in ('noise', 'test') else '%s\t%s'
            if sn not in ('noise', 'test'):
                set_lists[sn].insert(0, ('@FILE', 'STRING'))
            else:
                set_lists[sn].insert(0, '@FILE')
            np.savetxt(manifests[sn], set_lists[sn], fmt=format_str)

    return [manifests[sn] for sn in set_names]
def ingest_whales ( input_dir , out_dir , overwrite = False , train_frac = 0.8 ) : zfilename = os . path . join ( input_dir , 'whale_data.zip' ) orig_out_dir = out_dir out_dir = os . path . join ( orig_out_dir , 'whale-extracted' ) set_names = [ 'all' , 'val' , 'train' , 'noise' , 'test' ] manifests = { s : os . path . join ( out_dir , s + '-index.csv' ) for s in set_names } manifest_list_cfg = ', ' . join ( [ k + ':' + v for k , v in manifests . items ( ) ] ) cfg_path = os . path . dirname ( os . path . realpath ( __file__ ) ) for runtype in ( 'eval' , 'subm' ) : cfg_file = os . path . join ( cfg_path , 'whale_' + runtype + '.cfg' ) log_file = os . path . join ( orig_out_dir , 'train_' + runtype + '.log' ) with open ( cfg_file , 'w' ) as f : f . write ( 'manifest = [{}]\n' . format ( manifest_list_cfg ) ) f . write ( 'manifest_root = {}\n' . format ( out_dir ) ) f . write ( 'log = {}\n' . format ( log_file ) ) f . write ( 'epochs = 4\nrng_seed = 0\nverbose = True\n' ) if runtype == 'subm' : f . write ( 'save_path = {}\n' . format ( os . path . join ( orig_out_dir , 'model.p' ) ) ) f . write ( 'submission_file = {}\n' . format ( os . path . join ( orig_out_dir , 'subm.txt' ) ) ) if ( all ( [ os . path . exists ( manifests [ sn ] ) for sn in set_names ] ) and not overwrite ) : return [ manifests [ sn ] for sn in set_names ] with ZipFile ( zfilename , 'r' ) as zf : zf . extractall ( out_dir ) input_csv = os . path . join ( out_dir , 'data' , 'train.csv' ) train_records = np . genfromtxt ( input_csv , delimiter = ',' , skip_header = 1 , dtype = None ) np . random . seed ( 0 ) np . random . shuffle ( train_records ) pos_list , neg_list = [ ] , [ ] for aiff , lbl in train_records : try : record = ( os . path . join ( 'data' , 'train' , aiff ) , lbl ) except Exception : record = ( os . path . join ( 'data' , 'train' , aiff . decode ( ) ) , lbl ) if lbl == 1 : pos_list . append ( record ) else : neg_list . append ( record ) neg_part , pos_part = int ( len ( neg_list ) * train_frac ) , int ( len ( pos_list ) * train_frac ) set_lists = dict ( ) set_lists [ 'all' ] = neg_list + pos_list set_lists [ 'train' ] = neg_list [ : neg_part ] + pos_list [ : pos_part ] set_lists [ 'val' ] = neg_list [ neg_part : ] + pos_list [ pos_part : ] # Use just the files from the non-whale calls to use as a set of noise samples set_lists [ 'noise' ] = [ ( a ) for a , l in neg_list [ : neg_part ] ] # Write out the test files set_lists [ 'test' ] = [ os . path . relpath ( fn , out_dir ) for fn in glob ( os . path . join ( out_dir , 'data' , 'test' , '*.aiff' ) ) ] for sn in set_names : if sn != 'test' : np . random . shuffle ( set_lists [ sn ] ) format_str = '%s' if sn in ( 'noise' , 'test' ) else '%s\t%s' if sn not in ( 'noise' , 'test' ) : set_lists [ sn ] . insert ( 0 , ( '@FILE' , 'STRING' ) ) else : set_lists [ sn ] . insert ( 0 , '@FILE' ) np . savetxt ( manifests [ sn ] , set_lists [ sn ] , fmt = format_str ) return [ manifests [ sn ] for sn in set_names ]