Skip to content

Commit 83e9b5e

Browse files
MirceaDan99MirceaDan99
committed
POC for supporting blob metadata to be read from std::stream instead of std::vector<uint8_t> 
1 parent a2522be commit 83e9b5e

File tree

6 files changed

+154
-88
lines changed

6 files changed

+154
-88
lines changed

‎src/plugins/intel_npu/src/common/include/intel_npu/common/blob_container.hpp‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,29 +47,27 @@ class BlobContainerVector : public BlobContainer {
4747
class BlobContainerAlignedBuffer : public BlobContainer {
4848
public:
4949
BlobContainerAlignedBuffer(const std::shared_ptr<ov::AlignedBuffer>& blobSO,
50-
size_t ovHeaderOffset,
51-
size_t metadataSize)
52-
: _ownershipBlob(blobSO),
50+
size_t ovHeaderOffset, uint64_t blobSize)
51+
: _blobSize(blobSize),
5352
_ovHeaderOffset(ovHeaderOffset),
54-
_metadataSize(metadataSize) {}
53+
_ownershipBlob(blobSO) {}
5554

5655
void* get_ptr() override {
5756
return _ownershipBlob->get_ptr(_ovHeaderOffset);
5857
}
5958

6059
size_t size() const override {
61-
// remove OV header offset and metadata from blob size
62-
return _ownershipBlob->size() - _ovHeaderOffset - _metadataSize;
60+
return _blobSize;
6361
}
6462

6563
bool release_from_memory() override {
6664
return false;
6765
}
6866

6967
private:
70-
std::shared_ptr<ov::AlignedBuffer> _ownershipBlob;
68+
uint64_t _blobSize;
7169
size_t _ovHeaderOffset;
72-
size_t _metadataSize;
70+
std::shared_ptr<ov::AlignedBuffer> _ownershipBlob;
7371
};
7472

7573
} // namespace intel_npu

‎src/plugins/intel_npu/src/compiler_adapter/src/plugin_graph.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ size_t PluginGraph::export_blob(std::ostream& stream) const {
5151
_logger.info(str.str().c_str());
5252
}
5353
_logger.info("Write blob to stream successfully.");
54-
return _blob.size();
54+
return _blobPtr->size();
5555
}
5656

5757
std::vector<ov::ProfilingInfo> PluginGraph::process_profiling_output(const std::vector<uint8_t>& profData,

‎src/plugins/intel_npu/src/plugin/include/metadata.hpp‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
#include <string>
1212
#include <vector>
1313

14+
#include <openvino/runtime/aligned_buffer.hpp>
15+
1416
namespace intel_npu {
1517

1618
/**
@@ -99,6 +101,10 @@ struct MetadataBase {
99101

100102
virtual bool is_compatible() = 0;
101103

104+
virtual uint64_t get_blob_size() const = 0;
105+
106+
virtual size_t get_ov_header_offset() const = 0;
107+
102108
virtual ~MetadataBase() = default;
103109
};
104110

@@ -115,13 +121,17 @@ struct Metadata : public MetadataBase {};
115121
*/
116122
template <>
117123
struct Metadata<METADATA_VERSION_1_0> : public MetadataBase {
118-
protected:
124+
private:
119125
uint32_t _version;
120126
OpenvinoVersion _ovVersion;
127+
size_t _ovHeaderOffset;
128+
uint64_t _blobDataSize;
121129

122130
public:
123131
Metadata(std::optional<std::string_view> ovVersion = std::nullopt);
124132

133+
Metadata(size_t ovHeaderOffset, uint64_t blobDataSize);
134+
125135
void read(std::istream& stream) override;
126136

127137
void write(std::ostream& stream) override;
@@ -139,6 +149,14 @@ struct Metadata<METADATA_VERSION_1_0> : public MetadataBase {
139149
* @note The version check can be disabled if the "NPU_DISABLE_VERSION_CHECK" environment variable is set to '1'.
140150
*/
141151
bool is_compatible() override;
152+
153+
void set_version(uint32_t newVersion);
154+
155+
void set_ov_version(const OpenvinoVersion& newVersion);
156+
157+
uint64_t get_blob_size() const override;
158+
159+
size_t get_ov_header_offset() const override;
142160
};
143161

144162
/**
@@ -147,14 +165,16 @@ struct Metadata<METADATA_VERSION_1_0> : public MetadataBase {
147165
* @return Unique pointer to the created MetadataBase object if the major version is supported; otherwise, returns
148166
* 'nullptr'.
149167
*/
150-
std::unique_ptr<MetadataBase> create_metadata(uint32_t version);
168+
std::unique_ptr<MetadataBase> create_metadata(uint32_t version, size_t ovHeaderOffset, uint64_t blobDataSize);
151169

152170
/**
153171
* @brief Reads metadata from a blob.
154172
*
155173
* @return If the blob is versioned and its major version is supported, returns an unique pointer to the read
156174
* MetadataBase object; otherwise, returns 'nullptr'.
157175
*/
158-
std::unique_ptr<MetadataBase> read_metadata_from(const std::vector<uint8_t>& blob);
176+
std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream);
177+
178+
std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream, const std::shared_ptr<ov::AlignedBuffer>& modelBuffer);
159179

160180
} // namespace intel_npu

‎src/plugins/intel_npu/src/plugin/src/metadata.cpp‎

Lines changed: 100 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,35 @@
1111
#include "intel_npu/config/config.hpp"
1212
#include "intel_npu/utils/logger/logger.hpp"
1313
#include "openvino/core/version.hpp"
14+
#include "openvino/runtime/shared_buffer.hpp"
15+
16+
namespace {
17+
18+
size_t getFileSize(std::istream& stream) {
19+
auto log = intel_npu::Logger::global().clone("getFileSize");
20+
if (!stream) {
21+
OPENVINO_THROW("Stream is in bad status! Please check the passed stream status!");
22+
}
23+
24+
const size_t streamStart = stream.tellg();
25+
stream.seekg(0, std::ios_base::end);
26+
const size_t streamEnd = stream.tellg();
27+
stream.seekg(streamStart, std::ios_base::beg);
28+
29+
log.debug("Read blob size: streamStart=%zu, streamEnd=%zu", streamStart, streamEnd);
30+
31+
if (streamEnd < streamStart) {
32+
OPENVINO_THROW("Invalid stream size: streamEnd (",
33+
streamEnd,
34+
") is not larger than streamStart (",
35+
streamStart,
36+
")!");
37+
}
38+
39+
return streamEnd - streamStart;
40+
}
41+
42+
} // namespace
1443

1544
namespace intel_npu {
1645

@@ -29,9 +58,11 @@ void OpenvinoVersion::write(std::ostream& stream) {
2958
stream.write(_version.data(), _size);
3059
}
3160

32-
Metadata<METADATA_VERSION_1_0>::Metadata(std::optional<std::string_view> ovVersion)
61+
Metadata<METADATA_VERSION_1_0>::Metadata(std::optional<std::string_view> ovVersion, size_t ovHeaderOffset, uint64_t blobDataSize)
3362
: _version{METADATA_VERSION_1_0},
34-
_ovVersion{ovVersion.value_or(ov::get_openvino_version().buildNumber)} {}
63+
_ovVersion{ovVersion.value_or(ov::get_openvino_version().buildNumber)},
64+
_ovHeaderOffset{ovHeaderOffset},
65+
_blobDataSize{blobDataSize} {}
3566

3667
void Metadata<METADATA_VERSION_1_0>::read(std::istream& stream) {
3768
_ovVersion.read(stream);
@@ -42,13 +73,13 @@ void Metadata<METADATA_VERSION_1_0>::write(std::ostream& stream) {
4273
_ovVersion.write(stream);
4374
}
4475

45-
std::unique_ptr<MetadataBase> create_metadata(uint32_t version) {
76+
std::unique_ptr<MetadataBase> create_metadata(uint32_t version, size_t ovHeaderOffset, uint64_t blobDataSize) {
4677
switch (version) {
4778
case METADATA_VERSION_1_0:
48-
return std::make_unique<Metadata<METADATA_VERSION_1_0>>(std::nullopt);
79+
return std::make_unique<Metadata<METADATA_VERSION_1_0>>(std::nullopt, ovHeaderOffset, blobDataSize);
4980

5081
default:
51-
OPENVINO_THROW("Invalid metadata version!");
82+
return nullptr;
5283
}
5384
}
5485

@@ -76,44 +107,95 @@ bool Metadata<METADATA_VERSION_1_0>::is_compatible() {
76107
return true;
77108
}
78109

79-
std::unique_ptr<MetadataBase> read_metadata_from(const std::vector<uint8_t>& blob) {
110+
std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream) {
80111
Logger logger("NPUPlugin", Logger::global().level());
81112
size_t magicBytesSize = MAGIC_BYTES.size();
82113
std::string blobMagicBytes;
83114
blobMagicBytes.resize(magicBytesSize);
84115

85-
auto metadataIterator = blob.end() - magicBytesSize;
86-
std::memcpy(blobMagicBytes.data(), &(*metadataIterator), magicBytesSize);
116+
size_t currentStreamPos = stream.tellg();
117+
size_t streamSize = getFileSize(stream);
118+
stream.seekg(streamSize - magicBytesSize, std::ios::beg);
119+
stream.read(blobMagicBytes.data(), magicBytesSize);
87120
if (MAGIC_BYTES != blobMagicBytes) {
88-
OPENVINO_THROW("Blob is missing NPU metadata!");
121+
logger.error("Blob is missing NPU metadata!");
122+
return nullptr;
89123
}
90124

91125
uint64_t blobDataSize;
92-
metadataIterator -= sizeof(blobDataSize);
93-
std::memcpy(&blobDataSize, &(*metadataIterator), sizeof(blobDataSize));
94-
metadataIterator = blob.begin() + blobDataSize;
95-
96-
std::stringstream metadataStream;
97-
metadataStream.write(reinterpret_cast<const char*>(&(*metadataIterator)),
98-
blob.end() - metadataIterator - sizeof(blobDataSize));
126+
stream.seekg(streamSize - magicBytesSize - sizeof(blobDataSize), std::ios::beg);
127+
stream.read(reinterpret_cast<char*>(&blobDataSize), sizeof(blobDataSize));
128+
stream.seekg(currentStreamPos + blobDataSize, std::ios::beg);
99129

100130
uint32_t metaVersion;
101-
metadataStream.read(reinterpret_cast<char*>(&metaVersion), sizeof(metaVersion));
131+
stream.read(reinterpret_cast<char*>(&metaVersion), sizeof(metaVersion));
102132

103133
std::unique_ptr<MetadataBase> storedMeta;
104134
try {
105-
storedMeta = create_metadata(metaVersion);
135+
storedMeta = create_metadata(metaVersion, currentStreamPos, blobDataSize);
106136
storedMeta->read(metadataStream);
107137
} catch (...) {
108138
logger.warning("Imported blob metadata version: %d.%d, but the current version is: %d.%d",
109139
get_major(metaVersion),
110140
get_minor(metaVersion),
111141
get_major(CURRENT_METADATA_VERSION),
112142
get_minor(CURRENT_METADATA_VERSION));
143+
}
144+
stream.seekg(currentStreamPos, std::ios::beg);
145+
return storedMeta;
146+
}
147+
148+
std::unique_ptr<MetadataBase> read_metadata_from(std::istream& stream, const std::shared_ptr<ov::AlignedBuffer>& modelBuffer) {
149+
Logger logger("NPUPlugin", Logger::global().level());
150+
size_t magicBytesSize = MAGIC_BYTES.size();
151+
std::string blobMagicBytes;
152+
blobMagicBytes.resize(magicBytesSize);
153+
154+
size_t currentStreamPos = stream.tellg();
155+
size_t streamSize = modelBuffer->size();
156+
157+
blobMagicBytes.assign(reinterpret_cast<const char*>(modelBuffer->get_ptr(streamSize - magicBytesSize)), magicBytesSize);
158+
if (MAGIC_BYTES != blobMagicBytes) {
159+
logger.error("Blob is missing NPU metadata!");
160+
return nullptr;
161+
}
162+
163+
uint64_t blobDataSize;
164+
blobDataSize = *reinterpret_cast<uint64_t*>(modelBuffer->get_ptr(streamSize - magicBytesSize - sizeof(blobDataSize)));
165+
166+
uint32_t metaVersion;
167+
metaVersion = *reinterpret_cast<uint32_t*>(modelBuffer->get_ptr(currentStreamPos + blobDataSize));
168+
169+
stream.seekg(blobDataSize + sizeof(metaVersion), std::ios::cur);
170+
try {
171+
auto storedMeta = create_metadata(metaVersion, currentStreamPos, blobDataSize);
172+
storedMeta->read(stream);
173+
} catch(...) {
174+
logger.warning("Imported blob metadata version: %d.%d, but the current version is: %d.%d",
175+
get_major(metaVersion),
176+
get_minor(metaVersion),
177+
get_major(CURRENT_METADATA_VERSION),
178+
get_minor(CURRENT_METADATA_VERSION));
113179

114180
OPENVINO_THROW("NPU metadata mismatch.");
115181
}
116182
return storedMeta;
117183
}
118184

185+
void Metadata<METADATA_VERSION_1_0>::set_version(uint32_t newVersion) {
186+
_version = newVersion;
187+
}
188+
189+
void Metadata<METADATA_VERSION_1_0>::set_ov_version(const OpenvinoVersion& newVersion) {
190+
_ovVersion = newVersion;
191+
}
192+
193+
uint64_t Metadata<METADATA_VERSION_1_0>::get_blob_size() const {
194+
return _blobDataSize;
195+
}
196+
197+
size_t Metadata<METADATA_VERSION_1_0>::get_ov_header_offset() const {
198+
return _ovHeaderOffset;
199+
}
200+
119201
} // namespace intel_npu

‎src/plugins/intel_npu/src/plugin/src/plugin.cpp‎

Lines changed: 15 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -132,30 +132,6 @@ std::map<std::string, std::string> any_copy(const ov::AnyMap& params) {
132132
return result;
133133
}
134134

135-
size_t getFileSize(std::istream& stream) {
136-
auto log = Logger::global().clone("getFileSize");
137-
if (!stream) {
138-
OPENVINO_THROW("Stream is in bad status! Please check the passed stream status!");
139-
}
140-
141-
const size_t streamStart = stream.tellg();
142-
stream.seekg(0, std::ios_base::end);
143-
const size_t streamEnd = stream.tellg();
144-
stream.seekg(streamStart, std::ios_base::beg);
145-
146-
log.debug("Read blob size: streamStart=%zu, streamEnd=%zu", streamStart, streamEnd);
147-
148-
if (streamEnd < streamStart) {
149-
OPENVINO_THROW("Invalid stream size: streamEnd (",
150-
streamEnd,
151-
") is not larger than streamStart (",
152-
streamStart,
153-
")!");
154-
}
155-
156-
return streamEnd - streamStart;
157-
}
158-
159135
void update_log_level(const std::map<std::string, std::string>& propertiesMap) {
160136
auto it = propertiesMap.find(std::string(LOG_LEVEL::key()));
161137
if (it != propertiesMap.end()) {
@@ -784,10 +760,22 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, c
784760
auto compiler = compilerAdapterFactory.getCompiler(_backends->getIEngineBackend(), localConfig);
785761

786762
std::unique_ptr<BlobContainer> blobPtr;
763+
std::unique_ptr<MetadataBase> storedMeta;
787764

788-
if (modelBuffer == nullptr) {
789-
auto graphSize = getFileSize(stream);
765+
if (dynamic_cast<ov::OwningSharedStreamBuffer*>(stream.rdbuf())) {
766+
storedMeta = read_metadata_from(stream, dynamic_cast<ov::OwningSharedStreamBuffer*>(stream.rdbuf())->get_buffer());
767+
} else {
768+
storedMeta = read_metadata_from(stream);
769+
}
770+
771+
if (storedMeta == nullptr) {
772+
OPENVINO_THROW("Could not read metadata!");
773+
} else if (!storedMeta->is_compatible()) {
774+
OPENVINO_THROW("Incompatible blob version!");
775+
}
776+
auto graphSize = storedMeta->get_blob_size();
790777

778+
if (modelBuffer == nullptr) {
791779
std::vector<uint8_t> blob(graphSize);
792780
stream.read(reinterpret_cast<char*>(blob.data()), graphSize);
793781
if (!stream) {
@@ -797,7 +785,7 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, c
797785

798786
blobPtr = std::make_unique<BlobContainerVector>(std::move(blob));
799787
} else {
800-
blobPtr = std::make_unique<BlobContainerAlignedBuffer>(modelBuffer, stream.tellg(), 0);
788+
blobPtr = std::make_unique<BlobContainerAlignedBuffer>(modelBuffer, storedMeta->get_ov_header_offset(), graphSize);
801789
}
802790

803791
auto storedMeta = read_metadata_from(blob);

0 commit comments

Comments
 (0)