Fix Llama 3 TikToken conversion (#33538)

* Fix Llama 3 TikToken conversion

* No need to add tokens again

src/transformers/models/llama/convert_llama_weights_to_hf.py

@@ -332,7 +332,7 @@ def permute(w, n_heads, dim1=dim, dim2=dim):
 
 class Llama3Converter(TikTokenConverter):
     def __init__(self, vocab_file, special_tokens=None, instruct=False, model_max_length=None, **kwargs):
-        super().__init__(vocab_file, **kwargs)
+        super().__init__(vocab_file, additional_special_tokens=special_tokens, **kwargs)
         tokenizer = self.converted()
         chat_template = (
             "{% set loop_messages = messages %}"
@@ -345,7 +345,6 @@ def __init__(self, vocab_file, special_tokens=None, instruct=False, model_max_le
             "{% endfor %}"
             "{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"
         )
-        tokenizer.add_special_tokens(special_tokens)
 
         self.tokenizer = PreTrainedTokenizerFast(
             tokenizer_object=tokenizer,