[training] apply save_total_limit to push

[sanchit-gandhi]

Currently, the argument save_total_limit can be used to control the number of locally saved checkpoints, but not those pushed to the Hub. This PR ensures save_total_limit is applied to both (in-line with the HF Trainer). It was tested with the following training config:

librispeech_tts_r_300M_dummy.json

{
    "model_name_or_path": "./parler-tts-untrained-600M/parler-tts-untrained-600M/",
    "save_to_disk":  "./tmp_dataset_audio/",
    "temporary_save_to_disk": "./audio_code_tmp/",


    "feature_extractor_name":"ylacombe/dac_44khZ_8kbps",
    "description_tokenizer_name":"google/flan-t5-base",
    "prompt_tokenizer_name":"google/flan-t5-base",

    "report_to": ["wandb"],
    "overwrite_output_dir": true,
    "output_dir": "./parler-tts-save-total-limit",

    "train_dataset_name": "blabble-io/libritts_r",
    "train_metadata_dataset_name": "parler-tts/libritts_r_tags_tagged_10k_generated",
    "train_dataset_config_name": "clean",
    "train_split_name": "test.clean",

    "eval_dataset_name": "blabble-io/libritts_r",
    "eval_metadata_dataset_name": "parler-tts/libritts_r_tags_tagged_10k_generated",
    "eval_dataset_config_name": "clean",
    "eval_split_name": "test.clean",

    "target_audio_column_name": "audio", 
    "description_column_name": "text_description",
    "prompt_column_name": "text",

    "max_eval_samples": 48,
    "max_train_samples": 96,
    
    "max_duration_in_seconds": 20,
    "min_duration_in_seconds": 2.0,

    "add_audio_samples_to_wandb": true,
    "id_column_name": "id",

    "preprocessing_num_workers": 8,

    "do_train": true,
    "max_steps": 50,
    "gradient_accumulation_steps": 1,
    "gradient_checkpointing": false,
    "per_device_train_batch_size": 4,
    "learning_rate": 1e-3,
    "adam_beta1": 0.9,
    "adam_beta2": 0.99,
    "weight_decay": 0.01,

    "lr_scheduler_type": "cosine",
    "warmup_steps":  40,


    "logging_steps": 2,
    "freeze_text_encoder": true,


    "do_eval": false,
    "predict_with_generate": true,
    "include_inputs_for_metrics": true,
    "evaluation_strategy": "steps",
    "eval_steps": 50,
    "save_steps": 20,
    "push_to_hub": true,
    "save_total_limit": 1,

    "per_device_eval_batch_size": 12,

    "audio_encoder_per_device_batch_size":24,
    "dtype": "bfloat16",
    "seed": 456,

    "dataloader_num_workers":8
}

Which gives a repo where the checkpoints are rotated upon each save step, e.g. sanchit-gandhi/parler-tts-save-total-limit

The PR also saves the model config and generation config, such that a model can be loaded directly from a saved state.