From 3a8ea5604c2d68554085335d69e60854bd0f9747 Mon Sep 17 00:00:00 2001 From: Dwith Chenna Date: Fri, 23 Aug 2024 14:34:05 -0700 Subject: [PATCH] Merge branch 'dev' into unified_public --- tutorial/hello_world/hello_world.ipynb | 182 ++++++++++++++++--------- 1 file changed, 118 insertions(+), 64 deletions(-) diff --git a/tutorial/hello_world/hello_world.ipynb b/tutorial/hello_world/hello_world.ipynb index b1713a8c..98c0c4de 100644 --- a/tutorial/hello_world/hello_world.ipynb +++ b/tutorial/hello_world/hello_world.ipynb @@ -8,59 +8,31 @@ "\n", "This is a simple Jupyter Notebook that walks through the 4 steps of compiling and running a PyTorch model on the embedded Neural Processing Unit (NPU) in your AMD Ryzen AI enabled PC. The steps are as follows:\n", "\n", - "1. Get model - download or create a PyTorch model that we will run on the NPU\n", - "2. Export to ONNX - convert the PyTorch model to ONNX format.\n", - "3. Quantize - optimize the model for faster inference on the NPU by reducing its precision to INT8.\n", - "4. Run Model on CPU and NPU - compare performance between running the model on the CPU and on the NPU." + "1. Get model\n", + "2. Export to ONNX\n", + "3. Quantize\n", + "4. Run Model on CPU and NPU" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Requirement already satisfied: torch in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from -r requirements.txt (line 1)) (2.4.0)\n", - "Requirement already satisfied: ipykernel in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from -r requirements.txt (line 2)) (6.29.5)\n", - "Requirement already satisfied: filelock in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (3.15.4)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (4.12.2)\n", - "Requirement already satisfied: sympy in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (1.13.2)\n", - "Requirement already satisfied: networkx in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (3.3)\n", - "Requirement already satisfied: jinja2 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (3.1.4)\n", - "Requirement already satisfied: fsspec in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (2024.6.1)\n", - "Requirement already satisfied: comm>=0.1.1 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (0.2.2)\n", - "Requirement already satisfied: debugpy>=1.6.5 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (1.8.5)\n", - "Requirement already satisfied: ipython>=7.23.1 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (8.26.0)\n", - "Requirement already satisfied: jupyter-client>=6.1.12 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (8.6.2)\n", - "Requirement already satisfied: jupyter-core!=5.0.*,>=4.12 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (5.7.2)\n", - "Requirement already satisfied: matplotlib-inline>=0.1 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (0.1.7)\n", - "Requirement already satisfied: nest-asyncio in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (1.6.0)\n", - "Requirement already satisfied: packaging in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (24.1)\n", - "Requirement already satisfied: psutil in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (6.0.0)\n", - "Requirement already satisfied: pyzmq>=24 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (26.1.0)\n", - "Requirement already satisfied: tornado>=6.1 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (6.4.1)\n", - "Requirement already satisfied: traitlets>=5.4.0 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipykernel->-r requirements.txt (line 2)) (5.14.3)\n", - "Requirement already satisfied: decorator in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (0.19.1)\n", - "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (3.0.47)\n", - "Requirement already satisfied: pygments>=2.4.0 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (2.18.0)\n", - "Requirement already satisfied: stack-data in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (0.6.2)\n", - "Requirement already satisfied: exceptiongroup in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (1.2.2)\n", - "Requirement already satisfied: colorama in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (0.4.6)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from jupyter-client>=6.1.12->ipykernel->-r requirements.txt (line 2)) (2.9.0)\n", - "Requirement already satisfied: platformdirs>=2.5 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel->-r requirements.txt (line 2)) (4.2.2)\n", - "Requirement already satisfied: pywin32>=300 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from jupyter-core!=5.0.*,>=4.12->ipykernel->-r requirements.txt (line 2)) (306)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from jinja2->torch->-r requirements.txt (line 1)) (2.1.5)\n", - "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from sympy->torch->-r requirements.txt (line 1)) (1.3.0)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.3 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from jedi>=0.16->ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (0.8.4)\n", - "Requirement already satisfied: wcwidth in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (0.2.13)\n", - "Requirement already satisfied: six>=1.5 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from python-dateutil>=2.8.2->jupyter-client>=6.1.12->ipykernel->-r requirements.txt (line 2)) (1.16.0)\n", - "Requirement already satisfied: executing>=1.2.0 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (2.0.1)\n", - "Requirement already satisfied: asttokens>=2.1.0 in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (2.4.1)\n", - "Requirement already satisfied: pure-eval in c:\\users\\vgods\\miniconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from stack-data->ipython>=7.23.1->ipykernel->-r requirements.txt (line 2)) (0.2.3)\n" + "Defaulting to user installation because normal site-packages is not writeable\n", + "Requirement already satisfied: torch in c:\\users\\vgods\\appdata\\roaming\\python\\python310\\site-packages (from -r requirements.txt (line 1)) (2.4.0)\n", + "Requirement already satisfied: filelock in c:\\users\\vgods\\appdata\\roaming\\python\\python310\\site-packages (from torch->-r requirements.txt (line 1)) (3.15.4)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in c:\\programdata\\anaconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (4.11.0)\n", + "Requirement already satisfied: sympy in c:\\programdata\\anaconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (1.13.2)\n", + "Requirement already satisfied: networkx in c:\\users\\vgods\\appdata\\roaming\\python\\python310\\site-packages (from torch->-r requirements.txt (line 1)) (3.3)\n", + "Requirement already satisfied: jinja2 in c:\\programdata\\anaconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from torch->-r requirements.txt (line 1)) (3.1.4)\n", + "Requirement already satisfied: fsspec in c:\\users\\vgods\\appdata\\roaming\\python\\python310\\site-packages (from torch->-r requirements.txt (line 1)) (2024.6.1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\programdata\\anaconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from jinja2->torch->-r requirements.txt (line 1)) (2.1.5)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\programdata\\anaconda3\\envs\\ryzen-ai-1.2.0\\lib\\site-packages (from sympy->torch->-r requirements.txt (line 1)) (1.3.0)\n" ] } ], @@ -75,7 +47,7 @@ "source": [ "### 0. Imports & Environment Variables\n", "\n", - "We'll use the following imports in our example. `torch` and `torch_nn` are used for building and running ML models. We'll use them to define a small neural network and to generate the model weights. `os` is used for interacting with the operating system and is used to manage our environment variables, file paths, and directories. `subprocess` allows us to retrieve the hardware information. `onnx` and `onnxruntime` are used to work with our model in the ONNX format and for running our inference. `vai_q_onnx` is part of the Vitis AI Quantizer for ONNX models. We use it to perform quantization, converting the model into an INT8 format that is optimized for the NPU." + "We'll use the following imports in our example." ] }, { @@ -100,7 +72,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As well, we want to set the environment variables based on the NPU device we have in our PC. For more information about NPU configurations, see: For more information about NPU configurations, refer to the official [AMD Ryzen AI Documentation](https://ryzenai.docs.amd.com/en/latest/runtime_setup.html)." + "As well, we want to set the environment variables based on the NPU device we have in our PC. For more information about NPU configurations, see: https://ryzenai.docs.amd.com/en/latest/runtime_setup.html" ] }, { @@ -192,6 +164,87 @@ "cell_type": "code", "execution_count": 7, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "APU Type: PHX/HPT\n" + ] + } + ], + "source": [ + "# Test code to figure out what system I'm on\n", + "def get_apu_info():\n", + " # Run pnputil as a subprocess to enumerate PCI devices\n", + " command = r'pnputil /enum-devices /bus PCI /deviceids '\n", + " process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n", + " stdout, stderr = process.communicate()\n", + " # Check for supported Hardware IDs\n", + " apu_type = ''\n", + " if 'PCI\\\\VEN_1022&DEV_1502&REV_00' in stdout.decode(): apu_type = 'PHX/HPT'\n", + " if 'PCI\\\\VEN_1022&DEV_17F0&REV_00' in stdout.decode(): apu_type = 'STX'\n", + " if 'PCI\\\\VEN_1022&DEV_17F0&REV_10' in stdout.decode(): apu_type = 'STX'\n", + " if 'PCI\\\\VEN_1022&DEV_17F0&REV_11' in stdout.decode(): apu_type = 'STX'\n", + " return apu_type\n", + "\n", + "apu_type = get_apu_info()\n", + "print(f\"APU Type: {apu_type}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Setting environment for PHX/HPT\n", + "XLNX_VART_FIRMWARE= C:\\Program Files\\RyzenAI\\1.2.0\\voe-4.0-win_amd64\\xclbins\\phoenix\\1x4.xclbin\n", + "NUM_OF_DPU_RUNNERS= 1\n", + "XLNX_TARGET_NAME= AMD_AIE2_Nx4_Overlay\n" + ] + } + ], + "source": [ + "def set_environment_variable(apu_type):\n", + "\n", + " install_dir = os.environ['RYZEN_AI_INSTALLATION_PATH']\n", + " match apu_type:\n", + " case 'PHX/HPT':\n", + " print(\"Setting environment for PHX/HPT\")\n", + " os.environ['XLNX_VART_FIRMWARE']= os.path.join(install_dir, 'voe-4.0-win_amd64', 'xclbins', 'phoenix', '1x4.xclbin')\n", + " os.environ['NUM_OF_DPU_RUNNERS']='1'\n", + " os.environ['XLNX_TARGET_NAME']='AMD_AIE2_Nx4_Overlay'\n", + " case 'STX':\n", + " print(\"Setting environment for STX\")\n", + " os.environ['XLNX_VART_FIRMWARE']= os.path.join(install_dir, 'voe-4.0-win_amd64', 'xclbins', 'strix', 'AMD_AIE2P_Nx4_Overlay.xclbin')\n", + " os.environ['NUM_OF_DPU_RUNNERS']='1'\n", + " os.environ['XLNX_TARGET_NAME']='AMD_AIE2_Nx4_Overlay'\n", + " case _:\n", + " print(\"Unrecognized APU type. Exiting.\")\n", + " exit()\n", + " print('XLNX_VART_FIRMWARE=', os.environ['XLNX_VART_FIRMWARE'])\n", + " print('NUM_OF_DPU_RUNNERS=', os.environ['NUM_OF_DPU_RUNNERS'])\n", + " print('XLNX_TARGET_NAME=', os.environ['XLNX_TARGET_NAME'])\n", + "\n", + "set_environment_variable(apu_type)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1. Get Model\n", + "Here, we'll use the PyTorch library to define and instantiate a simple neural network model called `SmallModel`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -256,7 +309,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -295,7 +348,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -308,7 +361,8 @@ "INFO:vai_q_onnx.quant_utils:Obtained calibration data with 1 iters\n", "INFO:vai_q_onnx.quantize:Removed initializers from input\n", "INFO:vai_q_onnx.quantize:Simplified model sucessfully\n", - "INFO:vai_q_onnx.quantize:Loading model...\n" + "INFO:vai_q_onnx.quantize:Loading model...\n", + "INFO:vai_q_onnx.quant_utils:The input ONNX model C:/Users/vgods/AppData/Local/Temp/vai.simp.av5354ht/model_simp.onnx can run inference successfully\n" ] }, { @@ -316,16 +370,16 @@ "output_type": "stream", "text": [ "[VAI_Q_ONNX_INFO]: Time information:\n", - "2024-08-23 10:12:35.362481\n", + "2024-08-16 10:37:50.999103\n", "[VAI_Q_ONNX_INFO]: OS and CPU information:\n", " system --- Windows\n", - " node --- vgodsoe-ryzen\n", + " node --- Mini-PC\n", " release --- 10\n", " version --- 10.0.26100\n", " machine --- AMD64\n", " processor --- AMD64 Family 25 Model 116 Stepping 1, AuthenticAMD\n", "[VAI_Q_ONNX_INFO]: Tools version information:\n", - " python --- 3.10.14\n", + " python --- 3.10.6\n", " onnx --- 1.16.2\n", " onnxruntime --- 1.17.0\n", " vai_q_onnx --- 1.17.0+511d6f4\n", @@ -365,14 +419,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:vai_q_onnx.quant_utils:The input ONNX model C:/Users/vgods/AppData/Local/Temp/vai.simp.kpf9kmm3/model_simp.onnx can run inference successfully\n", "INFO:vai_q_onnx.quantize:optimize the model for better hardware compatibility.\n", "INFO:vai_q_onnx.quantize:Start calibration...\n", "INFO:vai_q_onnx.quantize:Start collecting data, runtime depends on your model size and the number of calibration dataset.\n", "INFO:vai_q_onnx.calibrate:Finding optimal threshold for each tensor using PowerOfTwoMethod.MinMSE algorithm ...\n", "INFO:vai_q_onnx.calibrate:Use all calibration data to calculate min mse\n", - "Computing range: 100%|██████████| 10/10 [00:04<00:00, 2.30tensor/s]\n", - "INFO:vai_q_onnx.quantize:Finished the calibration of PowerOfTwoMethod.MinMSE which costs 4.6s\n", + "Computing range: 100%|██████████| 10/10 [00:02<00:00, 3.78tensor/s]\n", + "INFO:vai_q_onnx.quantize:Finished the calibration of PowerOfTwoMethod.MinMSE which costs 2.8s\n", "INFO:vai_q_onnx.qdq_quantizer:Remove QuantizeLinear & DequantizeLinear on certain operations(such as conv-relu).\n", "INFO:vai_q_onnx.refine:Adjust the quantize info to meet the compiler constraints\n" ] @@ -451,7 +504,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -489,7 +542,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -526,7 +579,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -567,15 +620,15 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "CPU Execution Time: 0.11257850000004055\n", - "NPU Execution Time: 0.08555689999997185\n" + "CPU Execution Time: 0.07306490000337362\n", + "NPU Execution Time: 0.06736009998712689\n" ] } ], @@ -601,7 +654,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -616,7 +669,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "And there you have it. Your first model running on the NPU. We recommend trying a more complex model like ResNet50 or a custom model to compare performance and accuracy on the NPU.\n" + "And there you have it. Your first model running on the NPU." ] } ], @@ -636,8 +689,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" - } + "version": "3.10.6" + }, + "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 4