Got Caught exception: bad_function_call during inference on GPU

[hermeschen1116]

Hello, I'm trying to run a self-converted version of Qwen2-VL 2B on an Intel GPU, but I keep getting an exception below while the same code works totally fine on a CPU. I've tried to trace the input of each step in the Python side but cannot figure out what the difference is between inference on a CPU and a GPU. Sometimes it'll works after I refresh my venv then another failure occurs the next time.

Here's my code (partial of the class to handle inference)

#load
from openvino import Core
from optimum.intel.openvino import OVModelForVisualCausalLM

device = kwargs.get("device", "CPU")
ov_core = Core()
ov_core.set_property(device, {"EXECUTION_MODE": "PERFORMANCE", "CACHE_DIR": ""})

processor = AutoProcessor.from_pretrained(self._path_or_hf_repo, use_fast=True, trust_remote_code=True)
model = OVModelForVisualCausalLM.from_pretrained(
    self._path_or_hf_repo,
    ov_config=kwargs.get("ov_config", {}),
    use_cache=kwargs.get("use_cache", True),
    trust_remote_code=True,
    device=device.lower(),
    export=False,
    compile=True,
)

#inference
prompt = self.processor.apply_chat_template(messages, add_generation_prompt=True, continue_final_message=False)
inputs = self.processor(text=prompt, images=image, return_tensors="pt")
generated_ids = self.model.generate(**inputs)
generated_texts = self.processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True)

with these ov_config

[load_config.ov_config]
KV_CACHE_PRECISION = "u8"
DYNAMIC_QUANTIZATION_GROUP_SIZE = "128" # will be commented out if test on GPU
PERFORMANCE_HINT = "LATENCY"

And I'm running this in Python 3.13.9 with the packages below. I also tried to run the code with the different composition of installed packages(also torch with different backend) and different model, but nothing changed.

nncf                      2.18.0
openvino                  2025.3.0
openvino-telemetry        2025.2.0
openvino-tokenizers       2025.3.0.0
optimum                   2.0.0
optimum-intel             1.26.1
optimum-onnx              0.0.3
pytorch-triton-xpu        3.5.0
torch                     2.9.0+xpu
torchvision               0.24.0+xpu
transformers              4.55.4

(BTW, sometimes openvino 2025.2.0 will throw another error that occured on both CPU and GPU.)

Caught exception: Exception from src/plugins/intel_gpu/src/runtime/ocl/ocl_stream.cpp:272:
E       [GPU] [CL_EXT] setArgUsm in KernelIntel failed, error code: -49 CL_INVALID_ARG_INDEX

And the error

src/vlm_server/internal/Model.py:285: in _response
    generated_ids = self.model.generate(**inputs)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.13/site-packages/torch/utils/_contextlib.py:123: in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.13/site-packages/transformers/generation/utils.py:2618: in generate
    result = self._sample(
.venv/lib/python3.13/site-packages/transformers/generation/utils.py:3602: in _sample
    outputs = self(**model_inputs, return_dict=True)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.13/site-packages/optimum/modeling_base.py:113: in __call__
    return self.forward(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.13/site-packages/optimum/intel/openvino/modeling_visual_language.py:2914: in forward
    result = super().forward(
.venv/lib/python3.13/site-packages/optimum/intel/openvino/modeling_visual_language.py:791: in forward
    return self.language_model.forward(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <optimum.intel.openvino.modeling_visual_language.OVModelWithEmbedForCausalLM object at 0x7caf96699fd0>, input_ids = None
attention_mask = tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1... 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1]])
past_key_values = None
position_ids = tensor([[[ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16,
          17, 18, 19, 20, 21, 22, 23, 24...9, 40, 41, 36, 37, 38, 39, 40, 41, 36, 37, 38, 39, 40, 41, 36, 37,
          38, 39, 40, 41, 42, 43, 44, 45, 46, 47]]])
inputs_embeds = tensor([[[-0.0075,  0.0098,  0.0053,  ..., -0.0015,  0.0090, -0.0060],
         [ 0.0023,  0.0172,  0.0163,  ...,  0.0...42,  0.0131,  ...,  0.0146,  0.0284, -0.0102],
         [ 0.0338, -0.0124,  0.0132,  ...,  0.0008, -0.0107,  0.0165]]])
kwargs = {'cache_position': tensor([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
        18, 19, 20,..., 67, 68, 69, 70, 71,
        72, 73, 74, 75, 76, 77]), 'return_dict': True, 'token_type_ids': None, 'use_cache': True}
inputs = {'attention_mask': array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1,...9, 40, 41, 36, 37, 38, 39, 40, 41, 36, 37, 38, 39,
         40, 41, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]]])}

    def forward(
        self,
        input_ids: torch.LongTensor,
        attention_mask: Optional[torch.LongTensor] = None,
        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
        position_ids: Optional[torch.LongTensor] = None,
        inputs_embeds: Optional[torch.LongTensor] = None,
        **kwargs,
    ):
        self.compile()

        inputs = self.prepare_inputs(
            input_ids=input_ids,
            attention_mask=attention_mask,
            past_key_values=past_key_values,
            position_ids=position_ids,
            inputs_embeds=inputs_embeds,
            **kwargs,
        )
        # Run inference
        self.request.start_async(inputs, share_inputs=True)
>       self.request.wait()
E       RuntimeError: Exception from src/inference/src/cpp/infer_request.cpp:245:
E       Exception from src/bindings/python/src/pyopenvino/core/infer_request.hpp:54:
E       Caught exception: bad_function_call

.venv/lib/python3.13/site-packages/optimum/intel/openvino/modeling_visual_language.py:223: RuntimeError

[rkazants]

@p-durandin, @Lyamin-Roman, could you please take a look?

[hermeschen1116]

Hello, @rkazants. It's being a long time but the problem still occurred so that I just want to check is there any progress on this?

[peterchen-intel]

@hermeschen1116 Can you please share followinginformation?

1. Is this still reproducible with latest OpenVINO nightly? https://docs.openvino.ai/2025/get-started/install-openvino.html?PACKAGE=OPENVINO_BASE&VERSION=NIGHTLY&OP_SYSTEM=WINDOWS&DISTRIBUTION=PIP
2. What does it mean "a self-converted version of Qwen2-VL 2B"?
3. Attach full reproducer, means it can be run directly with cmd "python reproducer.py" without guessing.
4. What are the OS, GPU driver version and GPU model?

[hermeschen1116]

Sorry for quite late reply.

1. I currently install nightly version. Below is what I installed.

 % uv pip list | rg "torch|openvino|optimum|transformer|nncf"
nncf                               2.19.0
openvino                           2026.1.0.dev20260203
openvino-telemetry                 2025.2.0
openvino-tokenizers                2026.1.0.0.dev20260203
optimum                            2.1.0.dev0
optimum-intel                      1.27.0.dev0+d49f6f6
optimum-onnx                       0.1.0.dev0
torch                              2.10.0+cpu
torchvision                        0.25.0+cpu
transformers                       4.57.6

2. I mean the model is converted from original model by myself.

3. Below is the reproducer. But actually it works normally on GPU. I tried different ov_config in my service. The error I got is different in each trial, such as INVALID_ARG_INDEX from openvino. Finally I found if I set CACHE_DIR="" and KV_CACHE_PRECISION="f16" (previously set to "u8") then everything works.

#!/usr/bin/env python
# reproducer.py
from __future__ import annotations

from PIL import Image
import cv2
import numpy
from openvino import Core
from openvino.properties import hint
from optimum.intel.openvino import OVModelForVisualCausalLM
from transformers import AutoProcessor


MODEL_ID = "EZCon/TLPR-ov"


def select_ov_device() -> str:
	core = Core()
	devices = core.available_devices
	# 為了重現 GPU 問題，優先選 GPU；如果沒有就直接丟錯
	for dev in devices:
		if "GPU" in dev:
			return "GPU"
	raise RuntimeError(f"No GPU device found in OpenVINO available_devices: {devices}")


def create_dummy_image(width: int = 160, height: int = 160) -> Image.Image:
	# 建一張簡單的 RGB 圖，避免 alpha channel 相關問題
	bgr = numpy.zeros((height, width, 3), dtype=numpy.uint8)
	bgr[:] = (0, 255, 0)  # 綠色
	rgb = cv2.cvtColor(bgr, cv2.COLOR_BGR2RGB)
	return Image.fromarray(numpy.ascontiguousarray(rgb, dtype=numpy.uint8))


def main() -> int:
	device = select_ov_device()
	print(f"Using OpenVINO device: {device}")

	core = Core()
	core.set_property(device, {hint.execution_mode: hint.ExecutionMode.PERFORMANCE})

	print(f"Loading processor and model from {MODEL_ID} ...")
	processor = AutoProcessor.from_pretrained(MODEL_ID, use_fast=False, trust_remote_code=True)

	# 這裡 ov_config 跟你 issue 裡保持一致，方便他們對照
	ov_config = {
		# "KV_CACHE_PRECISION": "u8",  # 如果要對比 CPU/GPU 行為，可以打開/關掉試
		# "DYNAMIC_QUANTIZATION_GROUP_SIZE": "128",  # 你原本 GPU 測試會註解掉
		"PERFORMANCE_HINT": "LATENCY"
	}

	model = OVModelForVisualCausalLM.from_pretrained(
		MODEL_ID,
		ov_config=ov_config,
		use_cache=True,
		trust_remote_code=True,
		device=device.lower(),  # "gpu"
		export=False,
		compile=True,
	)

	# 準備一個最簡單的「圖文」訊息
	messages = [{"role": "user", "content": [{"type": "text", "text": "Describe this image briefly."}, {"type": "image"}]}]

	image = [create_dummy_image(160, 160)]

	print("Building chat template and inputs ...")
	prompt = processor.apply_chat_template(messages, add_generation_prompt=True, continue_final_message=False)

	inputs = processor(text=prompt, images=image, return_tensors="pt")

	print("Running generate() on GPU ...")
	outputs = model.generate(**inputs)
	texts = processor.batch_decode(outputs, skip_special_tokens=True, clean_up_tokenization_spaces=True)
	print("Generated text:")
	print(texts[0])

	return 0


if __name__ == "__main__":
	raise SystemExit(main())

4. Here is my environment information:
   • OS: Ubuntu 26.04 “Resolute Raccoon” (development branch), kernel 6.18.0-9-generic
   • GPU model: Intel Arrow Lake-P [Intel Graphics] (rev 03)
   • GPU driver: in-kernel i915 DRM driver from Ubuntu kernel 6.18.0-9-generic