promptopt_config.yaml

# Specify one or more prompt refinement technique to be used. If you specify more than one prompt refinement techniques,
# all these technique would run on same seed data. Result, iterations needed & cost incurred for each of these
# technique would be logged. And winning technique for each data instance and overall would be logged.

# Supported prompt refinement techniques: Basic, RecursiveEval, MedPrompt
# Uncomment techniques that you want to use
############################ Critique Task Description Start ############################
prompt_technique_name: "critique_n_refine"
# unique_model_id of model defined in llm_config.yaml
unique_model_id: gpt-4o
# Number of iterations for conducting <mutation_rounds>  rounds of mutation of task description
# followed by refinement of instructions
mutate_refine_iterations: 3
# Number of rounds of mutation to be performed when generating different styles
mutation_rounds: 3
# Refine instruction post mutation
refine_instruction: true
# Number of iterations for refining task description and in context examples for few-shot
refine_task_eg_iterations: 3
# Number of variations of prompts to generate in given iteration
style_variation: 5
# Number of questions to be asked to LLM in a single batch, during training step
questions_batch_size: 1
# Number of batches of questions to correctly answered, for a prompt to be considered as performing good
min_correct_count: 3
# Max number of mini-batches on which we should evaluate our prompt
max_eval_batches: 6
# Number of top best performing prompts to be considered for next iterations
top_n: 1
# Description of task. This will be fed to prompt
task_description : 'Extract the second letter from the input word.'
# Base instruction, in line with your dataset. This will be fed to prompt
base_instruction : 'Output the second letter. Think step by step to arrive at the solution.'
# Instruction for specifying answer format
answer_format : 'For each input word, present the reasoning followed by the extracted letter (only single letter) between <ANS_START> and <ANS_END> tags'
# Number of samples from dataset, set aside as training data. In every iteration we would be drawing
# `questions_batch_size` examples from training data with replacement.
seen_set_size: 25
# Number of examples to be given for few shots
few_shot_count: 5
# Generate synthetic reasoning
generate_reasoning: true
# Generate description of an expert which can solve the task at hand
generate_expert_identity: true
# Generate keywords that describe the intent of the task
generate_intent_keywords: false
############################ Critique Task Description End ############################