diff --git a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py
index 6db55d536cc..076154490ac 100644
--- a/python/llm/src/ipex_llm/transformers/pipeline_parallel.py
+++ b/python/llm/src/ipex_llm/transformers/pipeline_parallel.py
@@ -32,7 +32,11 @@
 import asyncio
 import uuid
 import threading
-from pydantic import BaseModel
+try:
+    from pydantic import BaseModel
+except ImportError:
+    from abc import ABCMeta
+    BaseModel = ABCMeta
 
 # patch GenerationMixin.generate
 from transformers import GenerationMixin