Used cache position instead to get the seq len

src/transformers/models/phimoe/modeling_phimoe.py

@@ -1130,10 +1130,7 @@ def forward(
 
         hidden_states = inputs_embeds
 
-        kv_seq_len = hidden_states.shape[-2]
-        if past_key_values is not None:
-            kv_seq_len += past_key_values.get_usable_length(kv_seq_len)
-        position_embeddings = self.rotary_emb(hidden_states, seq_len=kv_seq_len)
+        position_embeddings = self.rotary_emb(hidden_states, seq_len=cache_position[-1] + 1)
 
         # decoder layers
         all_hidden_states = () if output_hidden_states else None