Skip to content

Commit c3dc39d

Browse files
javier-inteljavier-intel
committed
Catch model import failure and report the appropriate error
1 parent e634654 commit c3dc39d

File tree

4 files changed

+507
-410
lines changed

4 files changed

+507
-410
lines changed

‎onnxruntime/core/providers/openvino/backend_manager.cc‎

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "core/providers/openvino/ov_interface.h"
2121
#include "core/providers/openvino/ov_versions/capability.h"
2222
#include "core/providers/openvino/qdq_transformations/qdq_stripping.h"
23+
#include "core/providers/openvino/exceptions.h"
2324
#include "core/providers/openvino/qdq_transformations/qdq_scales_fix.h"
2425

2526
namespace onnxruntime {
@@ -154,30 +155,31 @@ BackendManager::BackendManager(SessionContext& session_context,
154155
subgraph_context_,
155156
shared_context_,
156157
model_stream);
157-
} catch (const OnnxRuntimeException& ex) {
158-
std::string exception_str = ex.what();
159-
160-
if (session_context_.device_type.find("NPU") != std::string::npos &&
161-
exception_str.find("intel_npu") != std::string::npos) {
162-
// Handle NPU device related errors
163-
#ifndef NDEBUG
164-
ORT_THROW(exception_str + "\nModel needs to be recompiled\n");
165-
#else
166-
std::string error_message = "UNKNOWN NPU ERROR";
167-
std::string error_code = "code 0x0";
168-
std::regex error_message_pattern(R"(\bZE_\w*\b)");
169-
std::regex error_code_pattern("code 0x[0-9a-fA-F]+");
170-
std::smatch matches;
171-
if (std::regex_search(exception_str, matches, error_message_pattern)) {
172-
error_message = matches[0];
173-
}
174-
if (std::regex_search(exception_str, matches, error_code_pattern)) {
175-
error_code = matches[0];
158+
} catch (const ovep_exception& ex) {
159+
#ifndef OPENVINO_DISABLE_NPU_FALLBACK
160+
bool eligible_for_cpu_fallback = device_type.find("NPU") != std::string::npos &&
161+
!session_context_.so_disable_cpu_ep_fallback &&
162+
!subgraph_context_.is_ep_ctx_graph;
163+
if (eligible_for_cpu_fallback) {
164+
std::string exception_str = ex.what();
165+
LOGS_DEFAULT(VERBOSE) << exception_str;
166+
LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU."
167+
<< "Falling back to OV CPU for execution";
168+
session_context_.device_type = "CPU";
169+
session_context_.precision = "FP32";
170+
try {
171+
concrete_backend_ = BackendFactory::MakeBackend(model_proto,
172+
session_context_,
173+
subgraph_context_,
174+
shared_context_,
175+
model_stream);
176+
} catch (std::string const& msg) {
177+
ORT_THROW(msg);
176178
}
177-
throw std::runtime_error(error_message + ", " + error_code + "\nModel needs to be recompiled\n");
179+
} else
178180
#endif
179-
} else {
180-
ORT_THROW(exception_str);
181+
{
182+
throw ex;
181183
}
182184
}
183185
}

‎onnxruntime/core/providers/openvino/exceptions.h‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Copyright (C) Intel Corporation
2+
// Licensed under the MIT License
3+
4+
#pragma once
5+
6+
#include <exception>
7+
#include <regex>
8+
#include <string>
9+
10+
#include "core/common/status.h"
11+
12+
namespace onnxruntime {
13+
namespace openvino_ep {
14+
15+
struct ovep_exception : public std::exception {
16+
enum class type {
17+
compile_model,
18+
import_model,
19+
query_prop,
20+
read_model,
21+
unknown,
22+
};
23+
24+
ovep_exception(const std::string& message,
25+
enum class type type) : message_{message},
26+
type_{type},
27+
error_code_{ze_result_code_from_string(message)},
28+
error_name_{ze_result_name_from_string(message)} {}
29+
30+
const char* what() const noexcept override {
31+
return message_.data();
32+
}
33+
34+
uint32_t get_code() const { return error_code_; }
35+
36+
operator common::Status() const {
37+
common::StatusCategory category_ort{common::ONNXRUNTIME};
38+
39+
if (type_ == type::unknown) {
40+
return {category_ort, common::FAIL, message_};
41+
}
42+
43+
// Newer drivers
44+
if ((type_ == type::import_model) &&
45+
(error_code_ == 0x7800000f /* ZE_RESULT_ERROR_INVALID_NATIVE_BINARY */)) {
46+
std::string message{error_name_ + ", code 0x" + std::to_string(error_code_) + "\nModel needs to be recompiled\n"};
47+
return {category_ort, common::INVALID_GRAPH, message};
48+
}
49+
50+
std::string error_message = "Unhandled exception type: " + std::to_string(static_cast<int>(type_));
51+
return {category_ort, common::FAIL, error_message};
52+
}
53+
54+
protected:
55+
std::string message_;
56+
type type_{type::unknown};
57+
uint32_t error_code_{0};
58+
std::string error_name_;
59+
60+
private:
61+
uint32_t ze_result_code_from_string(const std::string& ov_exception_string) {
62+
uint32_t error_code{0};
63+
std::regex error_code_pattern("code 0x([0-9a-fA-F]+)");
64+
std::smatch matches;
65+
if (std::regex_search(ov_exception_string, matches, error_code_pattern)) {
66+
std::from_chars(&(*matches[1].first), &(*matches[1].second), error_code, 16);
67+
}
68+
return error_code;
69+
}
70+
std::string ze_result_name_from_string(const std::string& ov_exception_string) {
71+
std::string error_message = "UNKNOWN NPU ERROR";
72+
std::regex error_message_pattern(R"(\bZE_\w*\b)");
73+
std::smatch matches;
74+
if (std::regex_search(ov_exception_string, matches, error_message_pattern)) {
75+
error_message = matches[0];
76+
}
77+
return error_message;
78+
}
79+
};
80+
81+
} // namespace openvino_ep
82+
} // namespace onnxruntime

‎onnxruntime/core/providers/openvino/openvino_execution_provider.cc‎

Lines changed: 60 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "core/providers/openvino/onnx_ctx_model_helper.h"
1313
#include "core/providers/openvino/ov_versions/capability.h"
1414
#include "core/providers/openvino/qdq_transformations/qdq_stripping.h"
15+
#include "core/providers/openvino/exceptions.h"
1516
#include "core/session/onnxruntime_session_options_config_keys.h"
1617
#include "openvino/core/version.hpp"
1718
#ifdef USE_OVEP_NPU_MEMORY
@@ -94,6 +95,7 @@ common::Status OpenVINOExecutionProvider::Compile(
9495
auto& logger = *GetLogger();
9596
Status status = Status::OK();
9697

98+
try {
9799
bool is_epctx_model = false;
98100
if (!fused_nodes.empty()) {
99101
// Assume these properties are constant for all the model subgraphs, otherwise move to SubGraphContext
@@ -127,65 +129,65 @@ common::Status OpenVINOExecutionProvider::Compile(
127129
file >> metadata;
128130
}
129131

130-
struct OpenVINOEPFunctionState {
131-
AllocateFunc allocate_func = nullptr;
132-
DestroyFunc destroy_func = nullptr;
133-
AllocatorHandle allocator_handle = nullptr;
134-
BackendManager& backend_manager;
135-
};
136-
137-
for (const FusedNodeAndGraph& fused_node_graph : fused_nodes) {
138-
const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph;
139-
const Node& fused_node = fused_node_graph.fused_node;
140-
141-
NodeComputeInfo compute_info;
142-
143-
// During backend creation, we check if user wants to use precompiled blob onnx model or the original model
144-
// For precompiled blob, directly load the model instead of compiling the model
145-
// For original model, check if the user wants to export a model with pre-compiled blob
146-
147-
auto& backend_manager = backend_managers_.emplace_back(session_context_,
148-
*shared_context_,
149-
fused_node,
150-
graph_body_viewer,
151-
logger,
152-
ep_ctx_handle_);
153-
154-
compute_info.create_state_func =
155-
[&backend_manager](ComputeContext* context, FunctionState* state) {
156-
OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState{
157-
.allocate_func = context->allocate_func,
158-
.destroy_func = context->release_func,
159-
.allocator_handle = context->allocator_handle,
160-
.backend_manager = backend_manager};
161-
*state = static_cast<FunctionState>(p);
162-
return 0;
163-
};
164-
165-
compute_info.compute_func = [](FunctionState state, const OrtApi* /* api */, OrtKernelContext* context) {
166-
auto function_state = static_cast<OpenVINOEPFunctionState*>(state);
167-
try {
168-
function_state->backend_manager.Compute(context);
169-
} catch (const std::exception& ex) {
170-
return common::Status(common::ONNXRUNTIME, common::FAIL, ex.what());
171-
}
172-
return Status::OK();
132+
struct OpenVINOEPFunctionState {
133+
AllocateFunc allocate_func = nullptr;
134+
DestroyFunc destroy_func = nullptr;
135+
AllocatorHandle allocator_handle = nullptr;
136+
BackendManager& backend_manager;
173137
};
174138

175-
compute_info.release_state_func =
176-
[](FunctionState state) {
177-
if (state) {
178-
OpenVINOEPFunctionState* function_state = static_cast<OpenVINOEPFunctionState*>(state);
179-
delete function_state;
180-
}
181-
};
139+
for (const FusedNodeAndGraph& fused_node_graph : fused_nodes) {
140+
const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph;
141+
const Node& fused_node = fused_node_graph.fused_node;
142+
143+
NodeComputeInfo compute_info;
144+
145+
// During backend creation, we check if user wants to use precompiled blob onnx model or the original model
146+
// For precompiled blob, directly load the model instead of compiling the model
147+
// For original model, check if the user wants to export a model with pre-compiled blob
148+
149+
auto& backend_manager = backend_managers_.emplace_back(session_context_,
150+
*shared_context_,
151+
fused_node,
152+
graph_body_viewer,
153+
logger,
154+
ep_ctx_handle_);
155+
156+
compute_info.create_state_func =
157+
[&backend_manager](ComputeContext* context, FunctionState* state) {
158+
OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState{
159+
.allocate_func = context->allocate_func,
160+
.destroy_func = context->release_func,
161+
.allocator_handle = context->allocator_handle,
162+
.backend_manager = backend_manager};
163+
*state = static_cast<FunctionState>(p);
164+
return 0;
165+
};
166+
167+
compute_info.compute_func = [](FunctionState state, const OrtApi* /* api */, OrtKernelContext* context) {
168+
auto function_state = static_cast<OpenVINOEPFunctionState*>(state);
169+
try {
170+
function_state->backend_manager.Compute(context);
171+
} catch (const std::exception& ex) {
172+
return common::Status(common::ONNXRUNTIME, common::FAIL, ex.what());
173+
}
174+
return Status::OK();
175+
};
182176

183-
node_compute_funcs.push_back(std::move(compute_info));
177+
compute_info.release_state_func =
178+
[](FunctionState state) {
179+
if (state) {
180+
OpenVINOEPFunctionState* function_state = static_cast<OpenVINOEPFunctionState*>(state);
181+
delete function_state;
182+
}
183+
};
184184

185-
if (!status.IsOK()) {
186-
break;
185+
node_compute_funcs.push_back(std::move(compute_info));
186+
187+
if (!status.IsOK()) {
188+
break;
189+
}
187190
}
188-
}
189191

190192
// The block below is executed during EP context model generation
191193
if (session_context_.so_context_enable &&
@@ -213,6 +215,10 @@ common::Status OpenVINOExecutionProvider::Compile(
213215
ORT_RETURN_IF_NOT(file, "Metadata file could not be written: ", metadata_file_path);
214216
file << metadata;
215217
}
218+
}
219+
catch (const ovep_exception& ex) {
220+
status = ex;
221+
}
216222

217223
return status;
218224
}

0 commit comments

Comments
 (0)