Skip to content

Commit 6e52fae

Browse files
preetha-intelpreetha-intel
committed
Add file handling with shared binary file
1 parent 27de00a commit 6e52fae

File tree

5 files changed

+469
-212
lines changed

5 files changed

+469
-212
lines changed

‎onnxruntime/core/providers/openvino/backend_manager.cc

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -216,16 +216,27 @@ Status BackendManager::ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphVie
216216
std::string model_blob_str;
217217
auto compiled_model = concrete_backend_->GetOVCompiledModel();
218218
if (session_context_.so_share_ep_contexts){
219-
std::ostringstream model_blob_stream;
220-
compiled_model.export_model(model_blob_stream);
219+
// std::ostringstream model_blob_stream;
220+
// compiled_model.export_model(model_blob_stream);
221221

222222
// std::ofstream file(metadata_filename, std::ios::app| std::ios::binary);
223223
// std::cout << " write to metadata bin - " << metadata_filename << std::endl;
224+
auto& subgraph_metadata = shared_context_.shared_weights.subgraph_metadata;
225+
226+
sw::SubgraphMetadata::Map::key_type key{subgraph_context_.subgraph_name};
227+
sw::SubgraphMetadata::Map::mapped_type value{};
228+
224229
auto& bin_file = shared_context_.shared_weights.shared_bin_file.bin_file_;
225230
if (bin_file.is_open()) {
226-
bin_file << model_blob_stream.str();
231+
// std::cout << "Current offset before "<< subgraph_context_.subgraph_name << " = " << bin_file.tellp() << std::endl;
232+
value.epctx_offset = bin_file.tellp();
233+
// bin_file << model_blob_stream.str();
234+
compiled_model.export_model(bin_file);
235+
// std::cout << "Current offset after "<< subgraph_context_.subgraph_name << " = " << bin_file.tellp() << std::endl;
236+
value.epctx_length = static_cast<size_t>(static_cast<std::streamoff>(bin_file.tellp()) - value.epctx_offset);
237+
// std::cout << "Key = " << key.name << " Offset = " << value.epctx_offset << " , length = " << value.epctx_length << std::endl;
238+
subgraph_metadata.emplace(key, std::move(value));
227239
}
228-
std::cout << "Current offset after "<< subgraph_context_.subgraph_name << " = " << bin_file.tellp() << std::endl;
229240

230241
model_blob_str = shared_context_.shared_weights.shared_bin_file.shared_bin_filename.filename().string();
231242
} else if (session_context_.so_context_embed_mode) {

‎onnxruntime/core/providers/openvino/backend_utils.cc

Lines changed: 0 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -19,153 +19,6 @@ using Exception = ov::Exception;
1919

2020
namespace onnxruntime {
2121
namespace openvino_ep {
22-
23-
SharedContext::SharedWeights::WeightsFile::WeightsFile(std::filesystem::path filename) : file_(filename, std::ios::in | std::ios::binary) {
24-
try {
25-
file_.exceptions(std::ifstream::failbit | std::ifstream::badbit);
26-
weights_size_ = file_.seekg(0, std::ios::end).tellg();
27-
} catch (std::ifstream::failure& e) {
28-
ORT_THROW("Error: Failed to open weight file at ", filename.string(), " ", e.what());
29-
}
30-
}
31-
32-
void SharedContext::SharedWeights::WeightsFile::load_weights(size_t file_offset, void* data, size_t size) {
33-
ORT_ENFORCE(file_offset < weights_size_ && size <= weights_size_ && (file_offset <= weights_size_ - size), "Error: File offset is out of bounds.");
34-
file_.seekg(file_offset);
35-
file_.read(reinterpret_cast<char*>(data), size);
36-
}
37-
38-
std::ostream& operator<<(std::ostream& stream, const SharedContext::SharedWeights::Metadata::Map& metadata) {
39-
try {
40-
stream << metadata.size();
41-
42-
// Write each key-value pair
43-
// Put elements in separate lines to facilitate reading
44-
for (const auto& [key, value] : metadata) {
45-
stream << std::endl
46-
<< key.name;
47-
stream << std::endl
48-
<< value.location;
49-
stream << std::endl
50-
<< value.data_offset;
51-
stream << std::endl
52-
<< value.size;
53-
stream << std::endl
54-
<< value.dimensions.size();
55-
for (const auto& dim : value.dimensions) {
56-
stream << std::endl
57-
<< dim;
58-
}
59-
stream << std::endl
60-
<< value.element_type;
61-
}
62-
} catch (const Exception& e) {
63-
ORT_THROW("Error: Failed to write map data.", e.what());
64-
} catch (...) {
65-
ORT_THROW("Error: Failed to write map data.");
66-
}
67-
68-
ORT_ENFORCE(stream.good(), "Error: Failed to write map data.");
69-
return stream;
70-
}
71-
72-
std::ostream& operator<<(std::ostream& stream,
73-
const SharedContext::SharedWeights::SubgraphMetadata::Map& subgraph_metadata) {
74-
try {
75-
stream << subgraph_metadata.size();
76-
77-
// Write each key-value pair
78-
// Put elements in separate lines to facilitate reading
79-
for (const auto& [key, value] : subgraph_metadata) {
80-
stream << std::endl
81-
<< key.name;
82-
stream << std::endl
83-
<< value.epctx_offset;
84-
stream << std::endl
85-
<< value.epctx_length;
86-
}
87-
} catch (const Exception& e) {
88-
ORT_THROW("Error: Failed to write subgraph map data.", e.what());
89-
} catch (...) {
90-
ORT_THROW("Error: Failed to write subgraph map data.");
91-
}
92-
ORT_ENFORCE(stream.good(), "Error: Failed to write subgraph map data.");
93-
return stream;
94-
}
95-
96-
std::istream& operator>>(std::istream& stream, SharedContext::SharedWeights::Metadata::Map& metadata) {
97-
size_t map_size{0};
98-
try {
99-
stream >> map_size;
100-
101-
while (!stream.eof()) {
102-
SharedContext::SharedWeights::Metadata::Key key;
103-
SharedContext::SharedWeights::Metadata::Value value;
104-
stream >> key.name;
105-
stream >> value.location;
106-
stream >> value.data_offset;
107-
stream >> value.size;
108-
size_t num_dimensions;
109-
stream >> num_dimensions;
110-
111-
if (stream.fail()) {
112-
ORT_THROW("Error: Failed to read num_dimensions from stream.");
113-
}
114-
115-
constexpr size_t MAX_SAFE_DIMENSIONS = 1024;
116-
117-
size_t safe_num_dimensions = num_dimensions;
118-
119-
if (num_dimensions == 0 || safe_num_dimensions > MAX_SAFE_DIMENSIONS) {
120-
ORT_THROW("Invalid number of dimensions provided.");
121-
}
122-
try {
123-
value.dimensions.resize(safe_num_dimensions);
124-
} catch (const std::bad_alloc&) {
125-
ORT_THROW("Error: Memory allocation failed while resizing dimensions.");
126-
}
127-
128-
for (auto& dim : value.dimensions) {
129-
stream >> dim;
130-
}
131-
stream >> value.element_type;
132-
metadata.emplace(key, value);
133-
}
134-
} catch (const Exception& e) {
135-
ORT_THROW("Error: Failed to read map data.", e.what());
136-
} catch (...) {
137-
ORT_THROW("Error: Failed to read map data.");
138-
}
139-
140-
ORT_ENFORCE(metadata.size() == map_size, "Error: Inconsistent map data.");
141-
142-
return stream;
143-
}
144-
std::istream& operator>>(std::istream& stream, SharedContext::SharedWeights::SubgraphMetadata::Map& subgraph_metadata) {
145-
size_t map_size{0};
146-
try {
147-
stream >> map_size;
148-
149-
while (!stream.eof()) {
150-
SharedContext::SharedWeights::SubgraphMetadata::Key key;
151-
SharedContext::SharedWeights::SubgraphMetadata::Value value;
152-
stream >> key.name;
153-
stream >> value.epctx_offset;
154-
stream >> value.epctx_length;
155-
156-
subgraph_metadata.emplace(key, value);
157-
}
158-
} catch (const Exception& e) {
159-
ORT_THROW("Error: Failed to read map data.", e.what());
160-
} catch (...) {
161-
ORT_THROW("Error: Failed to read map data.");
162-
}
163-
164-
ORT_ENFORCE(subgraph_metadata.size() == map_size, "Error: Inconsistent map data.");
165-
166-
return stream;
167-
}
168-
16922
namespace backend_utils {
17023

17124
bool IsDebugEnabled() {

‎onnxruntime/core/providers/openvino/contexts.h

Lines changed: 60 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -27,20 +27,15 @@ class SharedContext : public WeakSingleton<SharedContext> {
2727
struct SharedWeights {
2828
struct Header {
2929
uint32_t bin_version=1;
30-
uint32_t footer_offset;
31-
Header(uint32_t bin_in, uint32_t footer_in) :
32-
bin_version(bin_in), footer_offset(footer_in){}
33-
};
30+
long footer_offset=0;
31+
}header_;
3432
struct Footer {
35-
uint32_t subgraph_offset;
36-
uint32_t subgraph_length;
37-
uint32_t metadata_offset;
38-
uint32_t metadata_length;
39-
Footer(uint32_t subgraph_offset_in, uint32_t subgraph_length_in,
40-
uint32_t metadata_offset_in, uint32_t metadata_length_in) :
41-
subgraph_offset(subgraph_offset_in), subgraph_length(subgraph_length_in),
42-
metadata_offset(metadata_offset_in), metadata_length(metadata_length_in) {}
43-
};
33+
long subgraph_offset;
34+
size_t subgraph_length;
35+
long metadata_offset;
36+
size_t metadata_length;
37+
}footer_;
38+
4439
struct Metadata {
4540
struct Key {
4641
std::string name;
@@ -60,8 +55,10 @@ class SharedContext : public WeakSingleton<SharedContext> {
6055
std::shared_ptr<ov::Tensor> tensor;
6156
};
6257
using Map = std::unordered_map<Key, Value, Hash>;
63-
friend std::ostream& operator<<(std::ostream& right, const Metadata::Map& metadata);
64-
friend std::istream& operator>>(std::istream& right, Metadata::Map& metadata);
58+
void writeMetadataToBinaryFile(SharedContext& shared_context, const Metadata::Map& metadata);
59+
void readMetadataFromBinaryFile(SharedContext& shared_context, Metadata::Map& metadata);
60+
// friend std::ostream& operator<<(std::ostream& right, const Metadata::Map& metadata);
61+
// friend std::istream& operator>>(std::istream& right, Metadata::Map& metadata);
6562
};
6663

6764
struct SubgraphMetadata {
@@ -75,12 +72,16 @@ class SharedContext : public WeakSingleton<SharedContext> {
7572
}
7673
};
7774
struct Value {
78-
uint32_t epctx_offset;
79-
uint32_t epctx_length;
75+
long epctx_offset;
76+
size_t epctx_length;
8077
};
8178
using Map = std::unordered_map<Key, Value, Hash>;
82-
friend std::ostream& operator<<(std::ostream& right, const SubgraphMetadata::Map& subgraph_metadata);
83-
friend std::istream& operator>>(std::istream& right, SubgraphMetadata::Map& subgraph_metadata);
79+
void writeSubgraphDataToBinaryFile(SharedContext& shared_context,
80+
const SubgraphMetadata::Map& subgraph_metadata);
81+
void readSubgraphDataFromBinaryFile(SharedContext& shared_context,
82+
SubgraphMetadata::Map& subgraph_metadata);
83+
// friend std::ostream& operator<<(std::ostream& right, const SubgraphMetadata::Map& subgraph_metadata);
84+
// friend std::istream& operator>>(std::istream& right, SubgraphMetadata::Map& subgraph_metadata);
8485
};
8586

8687
struct WeightsFile {
@@ -96,22 +97,49 @@ class SharedContext : public WeakSingleton<SharedContext> {
9697
};
9798

9899
struct SharedBinFile {
99-
// ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(SharedBinFile);
100-
// SharedBinFile() = delete;
101-
// SharedBinFile(fs::path shared_bin_filename) :
102-
// bin_file_(shared_bin_filename, std::ios::out | std::ios::app| std::ios::binary) {
103-
// if(bin_file_.is_open())
104-
// std::cout << " Bin file opened " << std::endl;
105-
// }
106100
fs::path shared_bin_filename;
107-
std::ofstream bin_file_;
101+
std::fstream bin_file_;
102+
size_t bin_size_;
108103

109104
SharedBinFile() = default; // Default constructor
110-
~SharedBinFile() = default; // Prevent closing the file automatically
105+
~SharedBinFile() {
106+
if (bin_file_.is_open()) {
107+
bin_file_.close(); // Close file when object is destroyed
108+
}
109+
}
110+
111+
void openBinFile(const fs::path shared_bin_filename) {
112+
// Check if the file exists before trying to open
113+
if (!fs::exists(shared_bin_filename)) {
114+
std::cerr << "Error: The file does not exist at path: " << shared_bin_filename << std::endl;
115+
std::ofstream createFile(shared_bin_filename, std::ios::binary); // Create an empty binary file
116+
if (!createFile) {
117+
throw std::runtime_error("Failed to create the file!");
118+
}
119+
createFile.close();
120+
// throw std::runtime_error("Failed to open log file! File does not exist.");
121+
}
122+
123+
// Check if the file is accessible for reading and writing
124+
fs::perms file_perms = fs::status(shared_bin_filename).permissions();
125+
126+
if ((file_perms & fs::perms::owner_read) == fs::perms::none ||
127+
(file_perms & fs::perms::owner_write) == fs::perms::none) {
128+
std::cerr << "Error: Insufficient permissions for file: " << shared_bin_filename << std::endl;
129+
throw std::runtime_error("Failed to open log file! Insufficient permissions.");
130+
}
131+
111132

112-
void openBinFile(fs::path shared_bin_filename) {
113133
if (!bin_file_.is_open()) { // Prevent reopening
114-
bin_file_.open(shared_bin_filename, std::ios::out | std::ios::app | std::ios::binary);
134+
std::cout << " Bin file is not open " << std::endl;
135+
bin_file_.open(shared_bin_filename, std::ios::in | std::ios::out | std::ios::binary);
136+
std::cout << " bin file opened " << std::endl;
137+
bin_size_ = bin_file_.seekg(0, std::ios::end).tellg();
138+
139+
std::cout << " bin size = " << bin_size_ << std::endl;
140+
bin_file_.seekg(0, std::ios::beg); // Reset to the beginning of the file
141+
142+
115143
if (!bin_file_) {
116144
throw std::runtime_error("Failed to open log file!");
117145
}
@@ -121,10 +149,9 @@ class SharedContext : public WeakSingleton<SharedContext> {
121149

122150
fs::path external_weight_filename;
123151
std::unique_ptr<WeightsFile> mapped_weights;
124-
std::unique_ptr<Header> header_;
125-
std::unique_ptr<Footer> footer_;
126-
// std::unique_ptr<SharedBinFile> shared_bin_file;
152+
Metadata metadata_;
127153
Metadata::Map metadata;
154+
SubgraphMetadata subgraph_metadata_;
128155
SubgraphMetadata::Map subgraph_metadata;
129156
}shared_weights;
130157
};

0 commit comments

Comments
 (0)