Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup onnx test runner's code #23693

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft

Cleanup onnx test runner's code #23693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
312b404
Cleanup onnx test runner's code
snnn Feb 13, 2025
6920f53
format code
snnn Feb 13, 2025
d69f205
Add missing file
snnn Feb 13, 2025
cbd5180
Merge branch 'snnn/onnx_test_runner' of https://github.com/microsoft/…
snnn Feb 13, 2025
0a31129
revert
snnn Feb 13, 2025
f7f1fde
Merge remote-tracking branch 'origin/main' into snnn/onnx_test_runner
snnn Feb 14, 2025
6318916
Merge remote-tracking branch 'upstream/main' into snnn/onnx_test_runner
snnn Feb 14, 2025
e3e15ec
update
snnn Feb 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmake/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ endif()
set(CMAKE_C_STANDARD 99)

include(CheckCXXCompilerFlag)
if(NOT ANDROID)
include(CheckIncludeFile)
endif()
include(CheckLanguage)
include(CMakeDependentOption)
include(FetchContent)
Expand Down Expand Up @@ -734,6 +737,10 @@ if (WIN32)
endif()

else()
if(NOT ANDROID)
# On ANDROID it requires ANDROID API level >=28
check_include_file("glob.h" HAVE_GLOB_H)
endif()
check_cxx_compiler_flag(-Wambiguous-reversed-operator HAS_AMBIGUOUS_REVERSED_OPERATOR)
check_cxx_compiler_flag(-Wbitwise-instead-of-logical HAS_BITWISE_INSTEAD_OF_LOGICAL)
check_cxx_compiler_flag(-Wcast-function-type HAS_CAST_FUNCTION_TYPE)
Expand Down
2 changes: 1 addition & 1 deletion cmake/onnxruntime_config.h.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// Licensed under the MIT License.

#pragma once

#cmakedefine HAVE_GLOB_H
#cmakedefine HAS_BITWISE_INSTEAD_OF_LOGICAL
#cmakedefine HAS_CAST_FUNCTION_TYPE
#cmakedefine HAS_CATCH_VALUE
Expand Down
27 changes: 4 additions & 23 deletions cmake/onnxruntime_unittests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -808,7 +808,7 @@ if(NOT IOS)
onnxruntime_add_include_to_target(onnx_test_runner_common onnxruntime_common onnxruntime_framework
onnxruntime_test_utils onnx onnx_proto re2::re2 flatbuffers::flatbuffers Boost::mp11 safeint_interface)

add_dependencies(onnx_test_runner_common onnx_test_data_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
add_dependencies(onnx_test_runner_common ${onnxruntime_EXTERNAL_DEPENDENCIES})
target_include_directories(onnx_test_runner_common PRIVATE ${eigen_INCLUDE_DIRS}
${CMAKE_CURRENT_BINARY_DIR} ${ONNXRUNTIME_ROOT})

Expand Down Expand Up @@ -906,7 +906,6 @@ AddTest(
SOURCES ${all_tests} ${onnxruntime_unittest_main_src}
LIBS
${onnx_test_runner_common_lib} ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs}
onnx_test_data_proto
DEPENDS ${all_dependencies}
TEST_ARGS ${test_all_args}
)
Expand Down Expand Up @@ -1007,23 +1006,6 @@ endif()

set(test_data_target onnxruntime_test_all)

onnxruntime_add_static_library(onnx_test_data_proto ${TEST_SRC_DIR}/proto/tml.proto)
add_dependencies(onnx_test_data_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES})
#onnx_proto target should mark this definition as public, instead of private
target_compile_definitions(onnx_test_data_proto PRIVATE "-DONNX_API=")
onnxruntime_add_include_to_target(onnx_test_data_proto onnx_proto)
if (MSVC)
# Cutlass code has an issue with the following:
# warning C4100: 'magic': unreferenced formal parameter
target_compile_options(onnx_test_data_proto PRIVATE "/wd4100")
endif()
target_include_directories(onnx_test_data_proto PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
set_target_properties(onnx_test_data_proto PROPERTIES FOLDER "ONNXRuntimeTest")
if(NOT DEFINED onnx_SOURCE_DIR)
find_path(onnx_SOURCE_DIR NAMES "onnx/onnx-ml.proto3" "onnx/onnx-ml.proto" REQUIRED)
endif()
onnxruntime_protobuf_generate(APPEND_PATH IMPORT_DIRS ${onnx_SOURCE_DIR} TARGET onnx_test_data_proto)

#
# onnxruntime_ir_graph test data
#
Expand Down Expand Up @@ -1101,7 +1083,6 @@ endif()
set(onnx_test_libs
onnxruntime_test_utils
${ONNXRUNTIME_TEST_LIBS}
onnx_test_data_proto
${onnxruntime_EXTERNAL_LIBRARIES})

if (NOT IOS)
Expand Down Expand Up @@ -1201,7 +1182,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
target_compile_options(onnx_test_runner_common PRIVATE -D_CRT_SECURE_NO_WARNINGS)
endif()

if (NOT onnxruntime_REDUCED_OPS_BUILD AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
if (NOT onnxruntime_REDUCED_OPS_BUILD AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND onnx_SOURCE_DIR)
add_test(NAME onnx_test_pytorch_converted
COMMAND onnx_test_runner ${onnx_SOURCE_DIR}/onnx/backend/test/data/pytorch-converted)
add_test(NAME onnx_test_pytorch_operator
Expand Down Expand Up @@ -1258,7 +1239,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
#onnxruntime_common is kind of ok because it is thin, tiny and totally stateless.
set(onnxruntime_perf_test_libs
onnx_test_runner_common onnxruntime_test_utils onnxruntime_common
onnxruntime onnxruntime_flatbuffers onnx_test_data_proto
onnxruntime onnxruntime_flatbuffers
${onnxruntime_EXTERNAL_LIBRARIES}
${GETOPT_LIB_WIDE} ${SYS_PATH_LIB} ${CMAKE_DL_LIBS})
if(NOT WIN32)
Expand Down Expand Up @@ -1308,7 +1289,7 @@ if (NOT onnxruntime_ENABLE_TRAINING_TORCH_INTEROP)
if(WIN32)
target_link_libraries(onnxruntime_qnn_ctx_gen PRIVATE debug dbghelp advapi32)
endif()
target_link_libraries(onnxruntime_qnn_ctx_gen PRIVATE onnx_test_runner_common onnxruntime_test_utils onnxruntime_common onnxruntime_graph onnxruntime_session onnxruntime_providers onnxruntime_framework onnxruntime_util onnxruntime_mlas onnxruntime_optimizer onnxruntime_flatbuffers onnx_test_data_proto ${onnxruntime_test_providers_libs} ${onnxruntime_EXTERNAL_LIBRARIES} ${GETOPT_LIB_WIDE} ${SYS_PATH_LIB} ${CMAKE_DL_LIBS})
target_link_libraries(onnxruntime_qnn_ctx_gen PRIVATE onnx_test_runner_common onnxruntime_test_utils onnxruntime_common onnxruntime_graph onnxruntime_session onnxruntime_providers onnxruntime_framework onnxruntime_util onnxruntime_mlas onnxruntime_optimizer onnxruntime_flatbuffers ${onnxruntime_test_providers_libs} ${onnxruntime_EXTERNAL_LIBRARIES} ${GETOPT_LIB_WIDE} ${SYS_PATH_LIB} ${CMAKE_DL_LIBS})

set_target_properties(onnxruntime_qnn_ctx_gen PROPERTIES FOLDER "ONNXRuntimeTest")
endif()
Expand Down
108 changes: 0 additions & 108 deletions onnxruntime/test/ir/onnx_model_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,44 +26,6 @@ class ONNXModelsTest : public ::testing::Test {

std::unique_ptr<logging::Logger> logger_;
};
#ifdef ORT_RUN_EXTERNAL_ONNX_TESTS
// Tests that Resolve() properly clears the state of topological sorted nodes,
// inputs, outputs and valueInfo.
// Assumes the graph passed in has been previously resolved.
static void TestResolve(onnxruntime::Graph& graph) {
GraphViewer graph_viewer(graph);
auto& nodes_before = graph_viewer.GetNodesInTopologicalOrder();
auto& inputs_before = graph.GetInputs();
auto& outputs_before = graph.GetOutputs();
auto& value_info_before = graph.GetValueInfo();

// Touch the graph to force Resolve() to recompute.
graph.SetGraphResolveNeeded();
graph.SetGraphProtoSyncNeeded();
ASSERT_STATUS_OK(graph.Resolve());

GraphViewer graph_viewer_2(graph);
auto& nodes_after = graph_viewer_2.GetNodesInTopologicalOrder();
auto& inputs_after = graph.GetInputs();
auto& outputs_after = graph.GetOutputs();
auto& value_info_after = graph.GetValueInfo();

// Multiple calls to Resolve() should not alter the sorted nodes,
// inputs, outputs and valueInfo. The internal state should be
// cleared.
EXPECT_EQ(nodes_before, nodes_after);
EXPECT_EQ(inputs_before, inputs_after);
EXPECT_EQ(outputs_before, outputs_after);
EXPECT_EQ(value_info_before, value_info_after);
}

TEST_F(ONNXModelsTest, squeeze_net) {
// NOTE: this requires the current directory to be where onnxruntime_ir_UT.exe is located
std::shared_ptr<Model> model;
ASSERT_STATUS_OK(Model::Load(ORT_TSTR("../models/opset8/test_squeezenet/model.onnx"), model, nullptr, *logger_));
TestResolve(model->MainGraph());
}
#endif

TEST_F(ONNXModelsTest, non_existing_model) {
// NOTE: this requires the current directory to be where onnxruntime_ir_UT.exe is located
Expand Down Expand Up @@ -96,76 +58,6 @@ class ONNXModelsTest1 : public ::testing::TestWithParam<const ORTCHAR_T*> {
return oss.str();
}
};
#ifdef ORT_RUN_EXTERNAL_ONNX_TESTS
TEST_F(ONNXModelsTest, bvlc_alexnet_1) {
using ::google::protobuf::io::CodedInputStream;
using ::google::protobuf::io::FileInputStream;
using ::google::protobuf::io::ZeroCopyInputStream;
int fd;
ASSERT_STATUS_OK(Env::Default().FileOpenRd(ORT_TSTR("../models/opset8/test_bvlc_alexnet/model.onnx"), fd));
ASSERT_TRUE(fd > 0);
std::unique_ptr<ZeroCopyInputStream> raw_input(new FileInputStream(fd));
std::unique_ptr<CodedInputStream> coded_input(new CodedInputStream(raw_input.get()));
// Allows protobuf library versions < 3.2.0 to parse messages greater than 64MB.
coded_input->SetTotalBytesLimit(INT_MAX);
ModelProto model_proto;
bool result = model_proto.ParseFromCodedStream(coded_input.get());
coded_input.reset();
raw_input.reset();
EXPECT_TRUE(result);
ASSERT_STATUS_OK(Env::Default().FileClose(fd));

std::shared_ptr<Model> model;
ASSERT_STATUS_OK(Model::Load(ORT_TSTR("../models/opset8/test_bvlc_alexnet/model.onnx"), model, nullptr,
*logger_));

// Check the graph input/output/value_info should have the same size as specified in the model file.
EXPECT_EQ(static_cast<size_t>(model_proto.graph().value_info_size()), model->MainGraph().GetValueInfo().size());
EXPECT_EQ(static_cast<size_t>(model_proto.graph().input_size()), model->MainGraph().GetInputs().size() + model->MainGraph().GetAllInitializedTensors().size());
EXPECT_EQ(static_cast<size_t>(model_proto.graph().output_size()), model->MainGraph().GetOutputs().size());
TestResolve(model->MainGraph());
}

TEST_P(ONNXModelsTest1, LoadFromFile) {
std::shared_ptr<Model> model;
ASSERT_STATUS_OK(Model::Load(GetModelFileName(), model, nullptr,
*logger_));
TestResolve(model->MainGraph());
}

TEST_P(ONNXModelsTest1, LoadFromProtobuf) {
using ::google::protobuf::io::CodedInputStream;
using ::google::protobuf::io::FileInputStream;
using ::google::protobuf::io::ZeroCopyInputStream;
int fd;
ASSERT_STATUS_OK(Env::Default().FileOpenRd(GetModelFileName(), fd));
ASSERT_TRUE(fd > 0);
std::unique_ptr<ZeroCopyInputStream> raw_input(new FileInputStream(fd));
std::unique_ptr<CodedInputStream> coded_input(new CodedInputStream(raw_input.get()));
coded_input->SetTotalBytesLimit(INT_MAX);
ModelProto model_proto;
bool result = model_proto.ParseFromCodedStream(coded_input.get());
coded_input.reset();
raw_input.reset();
ASSERT_TRUE(result);
ASSERT_STATUS_OK(Env::Default().FileClose(fd));
std::shared_ptr<Model> model;
ASSERT_STATUS_OK(Model::Load(std::move(model_proto), model, nullptr,
*logger_));
TestResolve(model->MainGraph());
}

#ifndef DISABLE_CONTRIB_OPS
INSTANTIATE_TEST_SUITE_P(ONNXModelsTests,
ONNXModelsTest1,
::testing::Values(ORT_TSTR("bvlc_alexnet"), ORT_TSTR("bvlc_googlenet"), ORT_TSTR("bvlc_reference_caffenet"), ORT_TSTR("bvlc_reference_rcnn_ilsvrc13"), ORT_TSTR("densenet121"), ORT_TSTR("emotion_ferplus"), ORT_TSTR("inception_v1"), ORT_TSTR("inception_v2"), ORT_TSTR("mnist"), ORT_TSTR("resnet50"), ORT_TSTR("shufflenet"), ORT_TSTR("squeezenet"), ORT_TSTR("tiny_yolov2"), ORT_TSTR("vgg19"), ORT_TSTR("zfnet512")));
#else
INSTANTIATE_TEST_SUITE_P(ONNXModelsTests,
ONNXModelsTest1,
::testing::Values(ORT_TSTR("bvlc_alexnet"), ORT_TSTR("bvlc_googlenet"), ORT_TSTR("bvlc_reference_caffenet"), ORT_TSTR("bvlc_reference_rcnn_ilsvrc13"), ORT_TSTR("densenet121"), ORT_TSTR("emotion_ferplus"), ORT_TSTR("inception_v1"), ORT_TSTR("inception_v2"), ORT_TSTR("mnist"), ORT_TSTR("resnet50"), ORT_TSTR("shufflenet"), ORT_TSTR("squeezenet"), ORT_TSTR("vgg19"), ORT_TSTR("zfnet512")));
#endif

#endif

// test a model that conforms to ONNX IR v4 where there are initializers that are not graph inputs.
// a NodeArg should be created for all initializers in this case.
Expand Down
Loading
Loading