gpt2-medium-persian / src /run_dataset.sh
m3hrdadfi's picture
Add dataset creation script
c92ce97
raw
history blame
327 Bytes
#!/bin/bash
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
export OUTPUT_DIR=/home/m3hrdadfi/data/
export DATASET_NAME=oscar
export DATASET_CONFIG_NAME=unshuffled_deduplicated_fa
python src/create_dataset.py \
--output_dir="$OUTPUT_DIR" \
--dataset_name="$DATASET_NAME" \
--dataset_config_name="$DATASET_CONFIG_NAME"