def get_dataset():
autotune = tf.data.experimental.AUTOTUNE
filenames = tf.data.Dataset.list_files(<path_to_files>+'/*',
shuffle=True)
def make_ds(file):
ds = arrow_io.ArrowFeatherDataset(
[file], [0,1],
output_types=(tf.uint8, tf.uint8),
output_shapes=([32*32*3,], []),
batch_mode='auto')
return ds
ds = filenames.interleave(make_ds,
num_parallel_calls=autotune,
deterministic=False)
ds = ds.unbatch()
return ds
Data Formats for Training in TensorFlow: Parquet, Petastorm, Feather, and More
