(filepattern, output_path)
return pipeline
Creates a glyphazzn dataset, from raw Parquetio to TFRecords.
Creates a glyphazzn dataset, from raw Parquetio to TFRecords.
Creates a glyphazzn dataset from raw Parquetio to TFRecords .
def create_glyphazzn_dataset(filepattern, output_path):
  """Creates a glyphazzn dataset, from raw Parquetio to TFRecords."""
  def pipeline(root):
    """Pipeline for creating glyphazzn dataset."""
    attrs = ['uni', 'width', 'vwidth', 'sfd', 'id', 'binary_fp']

    examples = root | 'Read' >> beam.io.parquetio.ReadFromParquet(
        file_pattern=filepattern, columns=attrs)

    examples = examples | 'FilterBadIcons' >> beam.Filter(_is_valid_glyph)
    examples = examples | 'ConvertToPath' >> beam.Map(_convert_to_path)
    examples = examples | 'FilterBadPathLenghts' >> beam.Filter(_is_valid_path)
    examples = examples | 'ProcessAndConvert' >> beam.Map(_create_example)
    (examples | 'WriteToTFRecord' >> beam.io.tfrecordio.WriteToTFRecord(
        output_path, num_shards=90))
  return pipeline
def create_glyphazzn_dataset ( filepattern , output_path ) : def pipeline ( root ) : """Pipeline for creating glyphazzn dataset.""" attrs = [ 'uni' , 'width' , 'vwidth' , 'sfd' , 'id' , 'binary_fp' ] examples = root | 'Read' >> beam . io . parquetio . ReadFromParquet ( file_pattern = filepattern , columns = attrs ) examples = examples | 'FilterBadIcons' >> beam . Filter ( _is_valid_glyph ) examples = examples | 'ConvertToPath' >> beam . Map ( _convert_to_path ) examples = examples | 'FilterBadPathLenghts' >> beam . Filter ( _is_valid_path ) examples = examples | 'ProcessAndConvert' >> beam . Map ( _create_example ) ( examples | 'WriteToTFRecord' >> beam . io . tfrecordio . WriteToTFRecord ( output_path , num_shards = 90 ) ) return pipeline