Skip to content

Commit

Permalink
Added FC precisions check in MatMulDecompressConvertTest
Browse files Browse the repository at this point in the history
  • Loading branch information
@v-Golubev
v-Golubev committed Jun 26, 2024
1 parent b0579f4 commit 2654ef9
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 77 deletions.
92 changes: 31 additions & 61 deletions ...lugins/intel_cpu/tests/functional/custom/subgraph_tests/src/matmul_decompress_convert.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,24 +100,11 @@ class MatMulDecompressConvertTest : public testing::WithParamInterface<MatMulDec
std::tie(inputShapes, transpose, weiElemType, additionalConfig, cpuParams) = obj.param;

std::ostringstream result;
for (const auto& shape : inputShapes) {
result << ov::test::utils::partialShape2str({shape.first}) << "_";
}
result << "TS=";
for (const auto& shape : inputShapes) {
result << "(";
if (!shape.second.empty()) {
auto itr = shape.second.begin();
do {
result << ov::test::utils::vec2str(*itr);
} while (++itr != shape.second.end() && result << "_");
}
result << ")_";
}
for (size_t i = 0; i < inputShapes.size(); ++i)
result << "IS[" << i << "]=" << inputShapes[i] << "_";
result << "transpose_a=" << transpose.first << "_";
result << "transpose_b=" << transpose.second << "_";

result << "weiLemType=" << weiElemType << "_";
result << "weiElemType=" << weiElemType << "_";

result << "config=(";
for (const auto& configEntry : additionalConfig) {
Expand All @@ -137,25 +124,6 @@ class MatMulDecompressConvertTest : public testing::WithParamInterface<MatMulDec
std::swap(*(shape.end() - 1), *(shape.end() - 2));
}

void check_fc_weights_precision(ElementType expectedWeiElemType) const {
auto getExecValue = [](const ov::Node::RTMap& rtInfo, const std::string& paramName) -> std::string {
auto it = rtInfo.find(paramName);
OPENVINO_ASSERT(rtInfo.end() != it);
return it->second.as<std::string>();
};

const auto execFunction = compiledModel.get_runtime_model();
ASSERT_NE(nullptr, execFunction);
for (const auto& fcNode : execFunction->get_ops()) {
if (getExecValue(fcNode->get_rt_info(), ov::exec_model_info::LAYER_TYPE) == "FullyConnected") {
const auto& constNode = fcNode->get_input_node_shared_ptr(1);
ov::element::Type expectedType(
getExecValue(constNode->get_rt_info(), ov::exec_model_info::OUTPUT_PRECISIONS));
ASSERT_EQ(expectedType, expectedWeiElemType);
}
}
}

void SetUp() override {
targetDevice = ov::test::utils::DEVICE_CPU;

Expand Down Expand Up @@ -196,23 +164,23 @@ class MatMulDecompressConvertTest : public testing::WithParamInterface<MatMulDec

configuration.insert(additionalConfig.begin(), additionalConfig.end());

ElementType netType = ElementType::f32;
ElementType convertOutType = ElementType::f32;
inType = outType = netType = ElementType::f32;
auto it = additionalConfig.find(ov::hint::inference_precision.name());
if (it != additionalConfig.end() && it->second.as<ov::element::Type>() == ov::element::bf16) {
convertOutType = inType = outType = netType = ElementType::bf16;
netType = ElementType::bf16;
weiConstElemType = (weiConstElemType != ElementType::f32) ? weiConstElemType : ElementType::bf16;
} else {
inType = outType = netType;
// Reorder between parameter and FullyConnected
// Note: reorder between FC and Result is not needed since FC primitive supports f32 output natively
reorderCount++;
}

std::string cpuNodeType = "FullyConnected";
selectedType = makeSelectedTypeStr(selectedType, outType);
selectedType = makeSelectedTypeStr(selectedType, netType);

ov::ParameterVector params{std::make_shared<ov::op::v0::Parameter>(inType, inShapeA)};
std::shared_ptr<ov::Node> inputB = ov::test::utils::make_constant(weiConstElemType, inShapeB.get_shape());
if (weiConstElemType == ElementType::f16 || weiConstElemType == ElementType::bf16) {
inputB = std::make_shared<ov::op::v0::Convert>(inputB, convertOutType);
if (weiConstElemType != inType) {
inputB = std::make_shared<ov::op::v0::Convert>(inputB, inType);
mark_as_decompression(inputB);
}
expectedWeiConstElemType = weiConstElemType;
Expand All @@ -223,17 +191,21 @@ class MatMulDecompressConvertTest : public testing::WithParamInterface<MatMulDec
}

void check_execution_graph() {
CheckNodePrecisionsWithType(compiledModel, "FullyConnected", {netType, expectedWeiConstElemType}, {outType});
CheckPluginRelatedResults(compiledModel, "FullyConnected");
CheckNumberOfNodesWithType(compiledModel, "FullyConnected", fullyConnectedCount);
CheckNumberOfNodesWithType(compiledModel, "Transpose", transposeCount);
CheckNumberOfNodesWithType(compiledModel, "Convert", 0);
CheckNumberOfNodesWithType(compiledModel, "Reorder", 0);
check_fc_weights_precision(expectedWeiConstElemType);
// Note: Convert node might be converted to Subgraph
CheckNumberOfNodesWithType(compiledModel, "Subgraph", 0);
CheckNumberOfNodesWithType(compiledModel, "Reorder", reorderCount);
}

size_t fullyConnectedCount = 1;
size_t transposeCount = 0;
size_t reorderCount = 0;
ElementType expectedWeiConstElemType = ElementType::f32;
ElementType netType = ElementType::f32;
};

TEST_P(MatMulDecompressConvertTest, CompareWithRefs) {
Expand Down Expand Up @@ -265,11 +237,9 @@ const std::vector<std::vector<InputShape>> inputShapes3D = {
{{{-1, -1, -1}, {{1, 2, 3}, {1, 5, 3}}}, {{1, 3, 4}, {{1, 3, 4}, {1, 3, 4}}}},
};

ov::AnyMap emptyConfig = {/* empty config */};

std::vector<ov::AnyMap> filter_additional_config_bf16() {
std::vector<ov::AnyMap> additionalConfig;
if (ov::with_cpu_x86_avx512_core()) {
if (ov::with_cpu_x86_bfloat16()) {
additionalConfig.push_back({{ov::hint::inference_precision(ov::element::bf16)}});
}
return additionalConfig;
Expand Down Expand Up @@ -303,7 +273,7 @@ std::vector<CPUSpecificParams> filter_specific_params_bf16() {
const auto testParams2D_FP32_smoke = ::testing::Combine(::testing::ValuesIn(inputShapes2D),
::testing::ValuesIn(transposeParams),
::testing::Values(ElementType::f32),
::testing::Values(emptyConfig),
::testing::Values(CPUTestUtils::empty_plugin_config),
::testing::ValuesIn(filter_specific_params(true)));

INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_FP32,
Expand All @@ -314,7 +284,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_FP32,
const auto testParams2D_smoke = ::testing::Combine(::testing::ValuesIn(inputShapes2D),
::testing::ValuesIn(transposeParams),
::testing::Values(ElementType::f16, ElementType::bf16),
::testing::Values(emptyConfig),
::testing::Values(CPUTestUtils::empty_plugin_config),
::testing::ValuesIn(filter_specific_params(false)));

INSTANTIATE_TEST_SUITE_P(smoke_FC_2D,
Expand All @@ -336,7 +306,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_BF16,
const auto testParams3D_FP32_smoke = ::testing::Combine(::testing::ValuesIn(inputShapes3D),
::testing::ValuesIn(transposeParams),
::testing::Values(ElementType::f32),
::testing::Values(emptyConfig),
::testing::Values(CPUTestUtils::empty_plugin_config),
::testing::ValuesIn(filter_specific_params(true)));

INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_FP32,
Expand All @@ -347,7 +317,7 @@ INSTANTIATE_TEST_SUITE_P(smoke_FC_3D_FP32,
const auto testParams3D_smoke = ::testing::Combine(::testing::ValuesIn(inputShapes3D),
::testing::ValuesIn(transposeParams),
::testing::Values(ElementType::f16, ElementType::bf16),
::testing::Values(emptyConfig),
::testing::Values(CPUTestUtils::empty_plugin_config),
::testing::ValuesIn(filter_specific_params(false)));

INSTANTIATE_TEST_SUITE_P(smoke_FC_3D,
Expand Down Expand Up @@ -461,26 +431,26 @@ class MatMulDecompressConvertTest2 : public MatMulDecompressConvertTest {

configuration.insert(additionalConfig.begin(), additionalConfig.end());

ElementType netType = ElementType::f32;
ElementType convertOutType = ElementType::f32;
inType = outType = netType = ElementType::f32;
auto it = additionalConfig.find(ov::hint::inference_precision.name());
if (it != additionalConfig.end() && it->second.as<ov::element::Type>() == ov::element::bf16) {
convertOutType = inType = outType = netType = ElementType::bf16;
netType = ElementType::bf16;
weiConstElemType = (weiConstElemType != ElementType::f32) ? weiConstElemType : ElementType::bf16;
} else {
inType = outType = netType;
// Reorder between parameter and FullyConnected
// Note: reorder between FC and Result is not needed since FC primitive supports f32 output natively
reorderCount++;
}

std::string cpuNodeType = "FullyConnected";
selectedType = makeSelectedTypeStr(selectedType, outType);
selectedType = makeSelectedTypeStr(selectedType, netType);

ov::ParameterVector params;
for (auto&& shape : {inShapeFC0, inShapeFC1}) {
params.push_back(std::make_shared<ov::op::v0::Parameter>(inType, shape));
}
std::shared_ptr<ov::Node> inputWeights = ov::test::utils::make_constant(weiConstElemType, inShapeWeights.get_shape());
if (weiConstElemType == ElementType::f16) {
inputWeights = std::make_shared<ov::op::v0::Convert>(inputWeights, convertOutType);
if (weiConstElemType != inType) {
inputWeights = std::make_shared<ov::op::v0::Convert>(inputWeights, inType);
mark_as_decompression(inputWeights);
}
expectedWeiConstElemType = weiConstElemType;
Expand All @@ -506,7 +476,7 @@ const auto testParams2D_FP16_2_smoke =
::testing::Combine(::testing::Values(static_shapes_to_test_representation({{2, 3}, {2, 3}, {3, 4}})),
::testing::Values(std::pair<bool, bool>{false, true}),
::testing::Values(ElementType::f16),
::testing::Values(emptyConfig),
::testing::Values(CPUTestUtils::empty_plugin_config),
::testing::ValuesIn(filter_specific_params(false)));

INSTANTIATE_TEST_SUITE_P(smoke_FC_2D_FP16_2,
Expand Down
44 changes: 28 additions & 16 deletions src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ ov::PrimitivesPriority CPUTestsBase::impls2primProiority(const std::vector<std::
return ov::PrimitivesPriority(str);
}

inline std::string getExecValue(const ov::Node::RTMap& rtInfo, const std::string& paramName) {
auto it = rtInfo.find(paramName);
OPENVINO_ASSERT(rtInfo.end() != it);
return it->second.as<std::string>();
}

void CPUTestsBase::CheckPluginRelatedResults(const ov::CompiledModel& execNet,
const std::set<std::string>& nodeType) const {
if (!execNet || nodeType.empty())
Expand All @@ -137,11 +143,6 @@ void CPUTestsBase::CheckPluginRelatedResultsImpl(const std::shared_ptr<const ov:
ASSERT_NE(nullptr, function);
for (const auto& node : function->get_ops()) {
const auto& rtInfo = node->get_rt_info();
auto getExecValue = [&rtInfo](const std::string& paramName) -> std::string {
auto it = rtInfo.find(paramName);
OPENVINO_ASSERT(rtInfo.end() != it);
return it->second.as<std::string>();
};
auto getExecValueOutputsLayout = [](const std::shared_ptr<ov::Node>& node) -> std::string {
auto rtInfo = node->get_rt_info();
auto it = rtInfo.find(ov::exec_model_info::OUTPUT_LAYOUTS);
Expand All @@ -162,7 +163,7 @@ void CPUTestsBase::CheckPluginRelatedResultsImpl(const std::shared_ptr<const ov:
return skip_unsquized_1D || permule_of_1;
};

if (nodeType.count(getExecValue(ov::exec_model_info::LAYER_TYPE))) {
if (nodeType.count(getExecValue(rtInfo, ov::exec_model_info::LAYER_TYPE))) {
ASSERT_LE(inFmts.size(), node->get_input_size());
ASSERT_LE(outFmts.size(), node->get_output_size());
for (size_t i = 0; i < inFmts.size(); i++) {
Expand Down Expand Up @@ -207,15 +208,15 @@ void CPUTestsBase::CheckPluginRelatedResultsImpl(const std::shared_ptr<const ov:
ASSERT_EQ(fmtsNum, actualOutputMemoryFormats.size());
}
for (size_t i = 0; i < fmtsNum; i++) {
const auto actualOutputMemoryFormat = getExecValue(ov::exec_model_info::OUTPUT_LAYOUTS);
const auto actualOutputMemoryFormat = getExecValue(rtInfo, ov::exec_model_info::OUTPUT_LAYOUTS);
const auto shape = node->get_output_partial_shape(i);

if (should_be_skipped(shape, outFmts[i]))
continue;
ASSERT_EQ(outFmts[i], cpu_str2fmt(actualOutputMemoryFormats[i].c_str()));
}

auto primType = getExecValue(ov::exec_model_info::IMPL_TYPE);
auto primType = getExecValue(rtInfo, ov::exec_model_info::IMPL_TYPE);

ASSERT_TRUE(primTypeCheck(primType))
<< "primType is unexpected : " << primType << " Expected : " << selectedType;
Expand Down Expand Up @@ -439,14 +440,7 @@ inline void CheckNumberOfNodesWithTypeImpl(std::shared_ptr<const ov::Model> func
ASSERT_NE(nullptr, function);
size_t actualNodeCount = 0;
for (const auto& node : function->get_ops()) {
const auto& rtInfo = node->get_rt_info();
auto getExecValue = [&rtInfo](const std::string& paramName) -> std::string {
auto it = rtInfo.find(paramName);
OPENVINO_ASSERT(rtInfo.end() != it);
return it->second.as<std::string>();
};

if (nodeTypes.count(getExecValue(ov::exec_model_info::LAYER_TYPE))) {
if (nodeTypes.count(getExecValue(node->get_rt_info(), ov::exec_model_info::LAYER_TYPE))) {
actualNodeCount++;
}
}
Expand All @@ -471,4 +465,22 @@ void CheckNumberOfNodesWithType(const ov::CompiledModel& compiledModel,
size_t expectedCount) {
CheckNumberOfNodesWithTypes(compiledModel, {nodeType}, expectedCount);
}

void CheckNodePrecisionsWithType(const ov::CompiledModel& compiledModel,
const std::string& nodeType,
ov::element::TypeVector&& inPrecisions,
ov::element::TypeVector&& outPrecisions) {
const auto function = compiledModel.get_runtime_model();
ASSERT_NE(nullptr, function);
for (const auto& node : function->get_ops()) {
if (getExecValue(node->get_rt_info(), ov::exec_model_info::LAYER_TYPE) == nodeType) {
ASSERT_EQ(inPrecisions.size(), node->get_input_size());
ASSERT_EQ(outPrecisions.size(), node->get_output_size());
for (size_t i = 0; i < inPrecisions.size(); ++i)
ASSERT_EQ(inPrecisions[i], node->get_input_element_type(i)) << "i = " << i;
for (size_t i = 0; i < outPrecisions.size(); ++i)
ASSERT_EQ(outPrecisions[i], node->get_output_element_type(i)) << "i = " << i;
}
}
}
} // namespace CPUTestUtils
5 changes: 5 additions & 0 deletions src/plugins/intel_cpu/tests/functional/utils/cpu_test_utils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,4 +182,9 @@ void CheckNumberOfNodesWithType(const ov::CompiledModel& compiledModel,
void CheckNumberOfNodesWithTypes(const ov::CompiledModel& compiledModel,
const std::unordered_set<std::string>& nodeTypes,
size_t expectedCount);

void CheckNodePrecisionsWithType(const ov::CompiledModel& compiledModel,
const std::string& nodeType,
ov::element::TypeVector&& inPrecisions,
ov::element::TypeVector&& outPrecisions);
} // namespace CPUTestUtils

0 comments on commit 2654ef9

Please sign in to comment.