Skip to content

Commit 358f9e7

Browse files
preetha-intelpreetha-intel
committed
Add file handling with shared binary file
1 parent fc08599 commit 358f9e7

File tree

5 files changed

+469
-212
lines changed

5 files changed

+469
-212
lines changed

‎onnxruntime/core/providers/openvino/backend_manager.cc

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -215,16 +215,27 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
215215
std::string model_blob_str;
216216
auto compiled_model = concrete_backend_->GetOVCompiledModel();
217217
if (session_context_.so_share_ep_contexts){
218-
std::ostringstream model_blob_stream;
219-
compiled_model.export_model(model_blob_stream);
218+
// std::ostringstream model_blob_stream;
219+
// compiled_model.export_model(model_blob_stream);
220220

221221
// std::ofstream file(metadata_filename, std::ios::app| std::ios::binary);
222222
// std::cout << " write to metadata bin - " << metadata_filename << std::endl;
223+
auto& subgraph_metadata = shared_context_.shared_weights.subgraph_metadata;
224+
225+
sw::SubgraphMetadata::Map::key_type key{subgraph_context_.subgraph_name};
226+
sw::SubgraphMetadata::Map::mapped_type value{};
227+
223228
auto& bin_file = shared_context_.shared_weights.shared_bin_file.bin_file_;
224229
if (bin_file.is_open()) {
225-
bin_file << model_blob_stream.str();
230+
// std::cout << "Current offset before "<< subgraph_context_.subgraph_name << " = " << bin_file.tellp() << std::endl;
231+
value.epctx_offset = bin_file.tellp();
232+
// bin_file << model_blob_stream.str();
233+
compiled_model.export_model(bin_file);
234+
// std::cout << "Current offset after "<< subgraph_context_.subgraph_name << " = " << bin_file.tellp() << std::endl;
235+
value.epctx_length = static_cast<size_t>(static_cast<std::streamoff>(bin_file.tellp()) - value.epctx_offset);
236+
// std::cout << "Key = " << key.name << " Offset = " << value.epctx_offset << " , length = " << value.epctx_length << std::endl;
237+
subgraph_metadata.emplace(key, std::move(value));
226238
}
227-
std::cout << "Current offset after "<< subgraph_context_.subgraph_name << " = " << bin_file.tellp() << std::endl;
228239

229240
model_blob_str = shared_context_.shared_weights.shared_bin_file.shared_bin_filename.filename().string();
230241
} else if (session_context_.so_context_embed_mode) {

‎onnxruntime/core/providers/openvino/backend_utils.cc

Lines changed: 0 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -19,153 +19,6 @@ using Exception = ov::Exception;
1919

2020
namespace onnxruntime {
2121
namespace openvino_ep {
22-
23-
SharedContext::SharedWeights::WeightsFile::WeightsFile(std::filesystem::path filename) : file_(filename, std::ios::in | std::ios::binary) {
24-
try {
25-
file_.exceptions(std::ifstream::failbit | std::ifstream::badbit);
26-
weights_size_ = file_.seekg(0, std::ios::end).tellg();
27-
} catch (std::ifstream::failure& e) {
28-
ORT_THROW("Error: Failed to open weight file at ", filename.string(), " ", e.what());
29-
}
30-
}
31-
32-
void SharedContext::SharedWeights::WeightsFile::load_weights(size_t file_offset, void* data, size_t size) {
33-
ORT_ENFORCE(file_offset < weights_size_ && size <= weights_size_ && (file_offset <= weights_size_ - size), "Error: File offset is out of bounds.");
34-
file_.seekg(file_offset);
35-
file_.read(reinterpret_cast<char*>(data), size);
36-
}
37-
38-
std::ostream& operator<<(std::ostream& stream, const SharedContext::SharedWeights::Metadata::Map& metadata) {
39-
try {
40-
stream << metadata.size();
41-
42-
// Write each key-value pair
43-
// Put elements in separate lines to facilitate reading
44-
for (const auto& [key, value] : metadata) {
45-
stream << std::endl
46-
<< key.name;
47-
stream << std::endl
48-
<< value.location;
49-
stream << std::endl
50-
<< value.data_offset;
51-
stream << std::endl
52-
<< value.size;
53-
stream << std::endl
54-
<< value.dimensions.size();
55-
for (const auto& dim : value.dimensions) {
56-
stream << std::endl
57-
<< dim;
58-
}
59-
stream << std::endl
60-
<< value.element_type;
61-
}
62-
} catch (const Exception& e) {
63-
ORT_THROW("Error: Failed to write map data.", e.what());
64-
} catch (...) {
65-
ORT_THROW("Error: Failed to write map data.");
66-
}
67-
68-
ORT_ENFORCE(stream.good(), "Error: Failed to write map data.");
69-
return stream;
70-
}
71-
72-
std::ostream& operator<<(std::ostream& stream,
73-
const SharedContext::SharedWeights::SubgraphMetadata::Map& subgraph_metadata) {
74-
try {
75-
stream << subgraph_metadata.size();
76-
77-
// Write each key-value pair
78-
// Put elements in separate lines to facilitate reading
79-
for (const auto& [key, value] : subgraph_metadata) {
80-
stream << std::endl
81-
<< key.name;
82-
stream << std::endl
83-
<< value.epctx_offset;
84-
stream << std::endl
85-
<< value.epctx_length;
86-
}
87-
} catch (const Exception& e) {
88-
ORT_THROW("Error: Failed to write subgraph map data.", e.what());
89-
} catch (...) {
90-
ORT_THROW("Error: Failed to write subgraph map data.");
91-
}
92-
ORT_ENFORCE(stream.good(), "Error: Failed to write subgraph map data.");
93-
return stream;
94-
}
95-
96-
std::istream& operator>>(std::istream& stream, SharedContext::SharedWeights::Metadata::Map& metadata) {
97-
size_t map_size{0};
98-
try {
99-
stream >> map_size;
100-
101-
while (!stream.eof()) {
102-
SharedContext::SharedWeights::Metadata::Key key;
103-
SharedContext::SharedWeights::Metadata::Value value;
104-
stream >> key.name;
105-
stream >> value.location;
106-
stream >> value.data_offset;
107-
stream >> value.size;
108-
size_t num_dimensions;
109-
stream >> num_dimensions;
110-
111-
if (stream.fail()) {
112-
ORT_THROW("Error: Failed to read num_dimensions from stream.");
113-
}
114-
115-
constexpr size_t MAX_SAFE_DIMENSIONS = 1024;
116-
117-
size_t safe_num_dimensions = num_dimensions;
118-
119-
if (num_dimensions == 0 || safe_num_dimensions > MAX_SAFE_DIMENSIONS) {
120-
ORT_THROW("Invalid number of dimensions provided.");
121-
}
122-
try {
123-
value.dimensions.resize(safe_num_dimensions);
124-
} catch (const std::bad_alloc&) {
125-
ORT_THROW("Error: Memory allocation failed while resizing dimensions.");
126-
}
127-
128-
for (auto& dim : value.dimensions) {
129-
stream >> dim;
130-
}
131-
stream >> value.element_type;
132-
metadata.emplace(key, value);
133-
}
134-
} catch (const Exception& e) {
135-
ORT_THROW("Error: Failed to read map data.", e.what());
136-
} catch (...) {
137-
ORT_THROW("Error: Failed to read map data.");
138-
}
139-
140-
ORT_ENFORCE(metadata.size() == map_size, "Error: Inconsistent map data.");
141-
142-
return stream;
143-
}
144-
std::istream& operator>>(std::istream& stream, SharedContext::SharedWeights::SubgraphMetadata::Map& subgraph_metadata) {
145-
size_t map_size{0};
146-
try {
147-
stream >> map_size;
148-
149-
while (!stream.eof()) {
150-
SharedContext::SharedWeights::SubgraphMetadata::Key key;
151-
SharedContext::SharedWeights::SubgraphMetadata::Value value;
152-
stream >> key.name;
153-
stream >> value.epctx_offset;
154-
stream >> value.epctx_length;
155-
156-
subgraph_metadata.emplace(key, value);
157-
}
158-
} catch (const Exception& e) {
159-
ORT_THROW("Error: Failed to read map data.", e.what());
160-
} catch (...) {
161-
ORT_THROW("Error: Failed to read map data.");
162-
}
163-
164-
ORT_ENFORCE(subgraph_metadata.size() == map_size, "Error: Inconsistent map data.");
165-
166-
return stream;
167-
}
168-
16922
namespace backend_utils {
17023

17124
bool IsDebugEnabled() {

‎onnxruntime/core/providers/openvino/contexts.h

Lines changed: 60 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -26,20 +26,15 @@ class SharedContext : public WeakSingleton<SharedContext> {
2626
struct SharedWeights {
2727
struct Header {
2828
uint32_t bin_version=1;
29-
uint32_t footer_offset;
30-
Header(uint32_t bin_in, uint32_t footer_in) :
31-
bin_version(bin_in), footer_offset(footer_in){}
32-
};
29+
long footer_offset=0;
30+
}header_;
3331
struct Footer {
34-
uint32_t subgraph_offset;
35-
uint32_t subgraph_length;
36-
uint32_t metadata_offset;
37-
uint32_t metadata_length;
38-
Footer(uint32_t subgraph_offset_in, uint32_t subgraph_length_in,
39-
uint32_t metadata_offset_in, uint32_t metadata_length_in) :
40-
subgraph_offset(subgraph_offset_in), subgraph_length(subgraph_length_in),
41-
metadata_offset(metadata_offset_in), metadata_length(metadata_length_in) {}
42-
};
32+
long subgraph_offset;
33+
size_t subgraph_length;
34+
long metadata_offset;
35+
size_t metadata_length;
36+
}footer_;
37+
4338
struct Metadata {
4439
struct Key {
4540
std::string name;
@@ -59,8 +54,10 @@ class SharedContext : public WeakSingleton<SharedContext> {
5954
std::shared_ptr<ov::Tensor> tensor;
6055
};
6156
using Map = std::unordered_map<Key, Value, Hash>;
62-
friend std::ostream& operator<<(std::ostream& right, const Metadata::Map& metadata);
63-
friend std::istream& operator>>(std::istream& right, Metadata::Map& metadata);
57+
void writeMetadataToBinaryFile(SharedContext& shared_context, const Metadata::Map& metadata);
58+
void readMetadataFromBinaryFile(SharedContext& shared_context, Metadata::Map& metadata);
59+
// friend std::ostream& operator<<(std::ostream& right, const Metadata::Map& metadata);
60+
// friend std::istream& operator>>(std::istream& right, Metadata::Map& metadata);
6461
};
6562

6663
struct SubgraphMetadata {
@@ -74,12 +71,16 @@ class SharedContext : public WeakSingleton<SharedContext> {
7471
}
7572
};
7673
struct Value {
77-
uint32_t epctx_offset;
78-
uint32_t epctx_length;
74+
long epctx_offset;
75+
size_t epctx_length;
7976
};
8077
using Map = std::unordered_map<Key, Value, Hash>;
81-
friend std::ostream& operator<<(std::ostream& right, const SubgraphMetadata::Map& subgraph_metadata);
82-
friend std::istream& operator>>(std::istream& right, SubgraphMetadata::Map& subgraph_metadata);
78+
void writeSubgraphDataToBinaryFile(SharedContext& shared_context,
79+
const SubgraphMetadata::Map& subgraph_metadata);
80+
void readSubgraphDataFromBinaryFile(SharedContext& shared_context,
81+
SubgraphMetadata::Map& subgraph_metadata);
82+
// friend std::ostream& operator<<(std::ostream& right, const SubgraphMetadata::Map& subgraph_metadata);
83+
// friend std::istream& operator>>(std::istream& right, SubgraphMetadata::Map& subgraph_metadata);
8384
};
8485

8586
struct WeightsFile {
@@ -95,22 +96,49 @@ class SharedContext : public WeakSingleton<SharedContext> {
9596
};
9697

9798
struct SharedBinFile {
98-
// ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SharedBinFile);
99-
// SharedBinFile() = delete;
100-
// SharedBinFile(fs::path shared_bin_filename) :
101-
// bin_file_(shared_bin_filename, std::ios::out | std::ios::app| std::ios::binary) {
102-
// if(bin_file_.is_open())
103-
// std::cout << " Bin file opened " << std::endl;
104-
// }
10599
fs::path shared_bin_filename;
106-
std::ofstream bin_file_;
100+
std::fstream bin_file_;
101+
size_t bin_size_;
107102

108103
SharedBinFile() = default; // Default constructor
109-
~SharedBinFile() = default; // Prevent closing the file automatically
104+
~SharedBinFile() {
105+
if (bin_file_.is_open()) {
106+
bin_file_.close(); // Close file when object is destroyed
107+
}
108+
}
109+
110+
void openBinFile(const fs::path shared_bin_filename) {
111+
// Check if the file exists before trying to open
112+
if (!fs::exists(shared_bin_filename)) {
113+
std::cerr << "Error: The file does not exist at path: " << shared_bin_filename << std::endl;
114+
std::ofstream createFile(shared_bin_filename, std::ios::binary); // Create an empty binary file
115+
if (!createFile) {
116+
throw std::runtime_error("Failed to create the file!");
117+
}
118+
createFile.close();
119+
// throw std::runtime_error("Failed to open log file! File does not exist.");
120+
}
121+
122+
// Check if the file is accessible for reading and writing
123+
fs::perms file_perms = fs::status(shared_bin_filename).permissions();
124+
125+
if ((file_perms & fs::perms::owner_read) == fs::perms::none ||
126+
(file_perms & fs::perms::owner_write) == fs::perms::none) {
127+
std::cerr << "Error: Insufficient permissions for file: " << shared_bin_filename << std::endl;
128+
throw std::runtime_error("Failed to open log file! Insufficient permissions.");
129+
}
130+
110131

111-
void openBinFile(fs::path shared_bin_filename) {
112132
if (!bin_file_.is_open()) { // Prevent reopening
113-
bin_file_.open(shared_bin_filename, std::ios::out | std::ios::app | std::ios::binary);
133+
std::cout << " Bin file is not open " << std::endl;
134+
bin_file_.open(shared_bin_filename, std::ios::in | std::ios::out | std::ios::binary);
135+
std::cout << " bin file opened " << std::endl;
136+
bin_size_ = bin_file_.seekg(0, std::ios::end).tellg();
137+
138+
std::cout << " bin size = " << bin_size_ << std::endl;
139+
bin_file_.seekg(0, std::ios::beg); // Reset to the beginning of the file
140+
141+
114142
if (!bin_file_) {
115143
throw std::runtime_error("Failed to open log file!");
116144
}
@@ -120,10 +148,9 @@ class SharedContext : public WeakSingleton<SharedContext> {
120148

121149
fs::path external_weight_filename;
122150
std::unique_ptr<WeightsFile> mapped_weights;
123-
std::unique_ptr<Header> header_;
124-
std::unique_ptr<Footer> footer_;
125-
// std::unique_ptr<SharedBinFile> shared_bin_file;
151+
Metadata metadata_;
126152
Metadata::Map metadata;
153+
SubgraphMetadata subgraph_metadata_;
127154
SubgraphMetadata::Map subgraph_metadata;
128155
}shared_weights;
129156
};

0 commit comments

Comments
 (0)