Skip to content

Commit 4ecaa5d

Browse files
olpipiolpipi
authored
Mmaped tensors (#29175)
### Details: - Add new ov::Tensor api to create tensor and read content from a file on disk . File can is read via mmap. - Add tests. ### Tickets: - CVS-162941
1 parent 523aa3e commit 4ecaa5d

File tree

4 files changed

+297
-1
lines changed

4 files changed

+297
-1
lines changed

‎src/core/dev_api/openvino/runtime/shared_buffer.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ template <typename T>
1313
class SharedBuffer : public ov::AlignedBuffer {
1414
public:
1515
SharedBuffer(char* data, size_t size, const T& shared_object) : _shared_object(shared_object) {
16-
m_allocated_buffer = data;
16+
m_allocated_buffer = nullptr;
1717
m_aligned_buffer = data;
1818
m_byte_size = size;
1919
}

‎src/core/include/openvino/runtime/tensor.hpp

+16
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,11 @@
99
*/
1010
#pragma once
1111

12+
#include <filesystem>
1213
#include <type_traits>
1314

1415
#include "openvino/core/coordinate.hpp"
16+
#include "openvino/core/partial_shape.hpp"
1517
#include "openvino/core/rtti.hpp"
1618
#include "openvino/core/shape.hpp"
1719
#include "openvino/core/type/element_type.hpp"
@@ -259,4 +261,18 @@ class OPENVINO_API Tensor {
259261
*/
260262
using TensorVector = std::vector<Tensor>;
261263

264+
/// \brief Read a tensor content from a file. Only raw data is loaded.
265+
/// \param file_name Path to file to read.
266+
/// \param element_type Element type, when not specified the it is assumed as element::u8.
267+
/// \param shape Shape for resulting tensor. If provided shape is static, specified number of elements is read only.
268+
/// File should contain enough bytes, an exception is raised otherwise.
269+
/// One of the dimensions can be dynamic. In this case it will be determined automatically based on the
270+
/// length of the file content and `offset`. Default value is [?].
271+
/// \param offset_in_bytes Read file starting from specified offset. Default is 0. The remining size of the file should
272+
/// be compatible with shape.
273+
OPENVINO_API
274+
Tensor read_tensor_data(const std::filesystem::path& file_name,
275+
const element::Type& element_type = element::u8,
276+
const PartialShape& shape = PartialShape::dynamic(1),
277+
std::size_t offset_in_bytes = 0);
262278
} // namespace ov

‎src/core/src/runtime/tensor.cpp

+76
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,13 @@
1111
#include "openvino/core/shape.hpp"
1212
#include "openvino/core/shape_util.hpp"
1313
#include "openvino/core/strides.hpp"
14+
#include "openvino/core/tensor_util.hpp"
15+
#include "openvino/core/type/element_iterator.hpp"
1416
#include "openvino/runtime/itensor.hpp"
1517
#include "openvino/runtime/make_tensor.hpp"
1618
#include "openvino/runtime/remote_tensor.hpp"
19+
#include "openvino/runtime/shared_buffer.hpp"
20+
#include "openvino/util/mmap_object.hpp"
1721

1822
namespace ov {
1923

@@ -108,4 +112,76 @@ bool Tensor::is_continuous() const {
108112
OV_TENSOR_STATEMENT(return _impl->is_continuous());
109113
}
110114

115+
namespace {
116+
ov::Shape calc_static_shape_for_file(const std::filesystem::path& file_name,
117+
const ov::element::Type& element_type,
118+
const ov::PartialShape& partial_shape,
119+
size_t offset) {
120+
auto file_size = std::filesystem::file_size(file_name);
121+
if (partial_shape.is_static()) {
122+
auto static_shape = partial_shape.get_shape();
123+
OPENVINO_ASSERT((ov::shape_size(static_shape)) * element_type.bitwidth() + offset * 8 == file_size * 8,
124+
"Cannot fit file size into requested static PartialShape");
125+
return static_shape;
126+
}
127+
auto partial_shape_copy = partial_shape;
128+
auto rank = partial_shape_copy.rank();
129+
OPENVINO_ASSERT(rank.is_static(), "Rank cannot be dynamic");
130+
std::vector<size_t> dynamic_dimension_numbers;
131+
size_t slice_size = 1;
132+
for (size_t id = 0; id < partial_shape_copy.size(); ++id) {
133+
if (partial_shape_copy[id].is_dynamic()) {
134+
dynamic_dimension_numbers.push_back(id);
135+
} else {
136+
slice_size *= partial_shape_copy[id].get_min_length();
137+
}
138+
}
139+
OPENVINO_ASSERT(dynamic_dimension_numbers.size() == 1,
140+
"Only one dynamic dimension in input shape is supported but got: ",
141+
dynamic_dimension_numbers.size());
142+
auto& dynamic_dimension = partial_shape_copy[dynamic_dimension_numbers[0]];
143+
144+
OPENVINO_ASSERT(file_size > offset, "Offset is bigger than size of file to read.");
145+
auto file_size_to_read = file_size - offset;
146+
147+
OPENVINO_ASSERT((file_size_to_read * 8) % element_type.bitwidth() == 0,
148+
"cannot fit ",
149+
element_type.get_type_name(),
150+
" into ",
151+
file_size_to_read,
152+
" bytes");
153+
auto elements_to_read = file_size_to_read * 8 / element_type.bitwidth();
154+
155+
auto new_dimension_size = elements_to_read / slice_size;
156+
OPENVINO_ASSERT(new_dimension_size * slice_size == elements_to_read,
157+
"Cannot fit file size into requested PartialShape");
158+
159+
OPENVINO_ASSERT(dynamic_dimension.compatible(new_dimension_size),
160+
"Cannot fit file size into requested PartialShape");
161+
162+
dynamic_dimension = Dimension(new_dimension_size);
163+
return partial_shape_copy.get_shape();
164+
}
165+
} // namespace
166+
167+
Tensor read_tensor_data(const std::filesystem::path& file_name,
168+
const ov::element::Type& element_type,
169+
const ov::PartialShape& partial_shape,
170+
size_t offset_in_bytes) {
171+
OPENVINO_ASSERT(element_type != ov::element::string);
172+
auto static_shape = calc_static_shape_for_file(file_name, element_type, partial_shape, offset_in_bytes);
173+
174+
auto mapped_memory = ov::load_mmap_object(file_name);
175+
auto shared_buffer =
176+
std::make_shared<ov::SharedBuffer<std::shared_ptr<ov::MappedMemory>>>(mapped_memory->data() + offset_in_bytes,
177+
mapped_memory->size() - offset_in_bytes,
178+
mapped_memory);
179+
180+
auto view_tensor = Tensor(element_type, static_shape, shared_buffer->get_ptr());
181+
auto impl = get_tensor_impl(view_tensor);
182+
impl._so = shared_buffer;
183+
view_tensor = make_tensor(impl);
184+
185+
return view_tensor;
186+
}
111187
} // namespace ov

‎src/core/tests/tensor_utils.cpp

+204
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
// Copyright (C) 2018-2025 Intel Corporation
2+
// SPDX-License-Identifier: Apache-2.0
3+
//
4+
5+
#include "common_test_utils/common_utils.hpp"
6+
#include "common_test_utils/data_utils.hpp"
7+
#include "openvino/core/type/element_iterator.hpp"
8+
#include "openvino/op/constant.hpp"
9+
#include "openvino/reference/convert.hpp"
10+
#include "openvino/runtime/tensor.hpp"
11+
12+
namespace ov::test {
13+
template <typename element_type>
14+
class ParametredOffloadTensorTest : public ::testing::Test {
15+
public:
16+
static constexpr ov::element::Type ov_type = ov::element::from<element_type>();
17+
18+
void SetUp() override {
19+
shape = {10, 20, 30, 40};
20+
auto static_shape = shape.get_shape();
21+
initial_tensor = Tensor(ov_type, static_shape);
22+
std::vector<float> init_values(initial_tensor.get_size());
23+
ov::test::utils::fill_data_random(init_values.data(), initial_tensor.get_size(), 10, 0, 100);
24+
25+
ov::reference::convert(ov::element::iterator<ov::element::f32>(init_values.data()),
26+
ov::element::iterator<ov_type>(initial_tensor.data()),
27+
shape_size(static_shape));
28+
29+
file_name = ov::test::utils::generateTestFilePrefix();
30+
data_size = this->initial_tensor.get_byte_size();
31+
{
32+
std::ofstream fout(this->file_name, std::ios::binary);
33+
fout.write(reinterpret_cast<char*>(this->initial_tensor.data()), data_size);
34+
}
35+
ASSERT_TRUE(std::filesystem::exists(this->file_name));
36+
}
37+
38+
void TearDown() override {
39+
remove_file();
40+
}
41+
42+
void remove_file() {
43+
if (std::filesystem::exists(file_name))
44+
std::filesystem::remove(file_name);
45+
}
46+
47+
ov::PartialShape shape;
48+
ov::Tensor initial_tensor;
49+
std::filesystem::path file_name;
50+
size_t data_size;
51+
};
52+
53+
TYPED_TEST_SUITE_P(ParametredOffloadTensorTest);
54+
55+
TYPED_TEST_P(ParametredOffloadTensorTest, read_tensor) {
56+
{
57+
ov::Tensor tensor;
58+
EXPECT_NO_THROW(tensor = read_tensor_data(this->file_name, this->ov_type, this->shape, 0));
59+
EXPECT_EQ(0, memcmp(tensor.data(), this->initial_tensor.data(), this->data_size));
60+
}
61+
}
62+
63+
REGISTER_TYPED_TEST_SUITE_P(ParametredOffloadTensorTest, read_tensor);
64+
65+
using TypesToTest = ::testing::Types<float,
66+
double,
67+
int8_t,
68+
int16_t,
69+
int32_t,
70+
int64_t,
71+
uint8_t,
72+
uint16_t,
73+
uint32_t,
74+
uint64_t,
75+
ov::bfloat16,
76+
ov::float8_e4m3,
77+
ov::float8_e5m2,
78+
ov::float4_e2m1,
79+
ov::float8_e8m0>;
80+
81+
INSTANTIATE_TYPED_TEST_SUITE_P(OffloadTensorTest, ParametredOffloadTensorTest, TypesToTest);
82+
83+
TEST(OffloadTensorTest, string_tensor_throws) {
84+
auto file_name = ov::test::utils::generateTestFilePrefix();
85+
{
86+
std::ofstream fout(file_name, std::ios::binary);
87+
fout << "Hello, world!";
88+
fout.close();
89+
ASSERT_TRUE(std::filesystem::exists(file_name));
90+
EXPECT_THROW(read_tensor_data(file_name, ov::element::string), ov::Exception);
91+
std::filesystem::remove(file_name);
92+
}
93+
}
94+
95+
class FunctionalOffloadTensorTest : public ::testing::Test {
96+
public:
97+
void SetUp() override {
98+
auto elements_number = ov::shape_size(shape.get_shape());
99+
data_size = elements_number * sizeof(float);
100+
init_values.resize(elements_number);
101+
ov::test::utils::fill_data_random(init_values.data(), elements_number, 10, 0, 100);
102+
103+
file_name = ov::test::utils::generateTestFilePrefix();
104+
{
105+
std::ofstream fout(file_name, std::ios::binary);
106+
fout.write(reinterpret_cast<char*>(init_values.data()), data_size);
107+
}
108+
ASSERT_TRUE(std::filesystem::exists(file_name));
109+
}
110+
111+
void TearDown() override {
112+
remove_file();
113+
}
114+
115+
void remove_file() {
116+
if (std::filesystem::exists(file_name))
117+
std::filesystem::remove(file_name);
118+
}
119+
120+
ov::element::Type ov_type{ov::element::f32};
121+
ov::PartialShape shape{1, 2, 3, 4};
122+
size_t data_size;
123+
std::vector<float> init_values;
124+
std::string file_name;
125+
};
126+
127+
TEST_F(FunctionalOffloadTensorTest, read_with_offset) {
128+
{
129+
float dummy = 0;
130+
std::ofstream fout(file_name, std::ios::binary);
131+
fout.write(reinterpret_cast<char*>(&dummy), sizeof(float));
132+
fout.write(reinterpret_cast<char*>(init_values.data()), data_size);
133+
}
134+
ASSERT_TRUE(std::filesystem::exists(file_name));
135+
136+
{
137+
ov::Tensor tensor;
138+
EXPECT_NO_THROW(tensor = read_tensor_data(file_name, ov_type, shape, sizeof(float)));
139+
EXPECT_EQ(0, memcmp(tensor.data(), init_values.data(), data_size));
140+
}
141+
}
142+
143+
TEST_F(FunctionalOffloadTensorTest, read_small_file) {
144+
auto new_shape = shape;
145+
new_shape[0] = 10;
146+
{ EXPECT_THROW(std::ignore = read_tensor_data(file_name, ov_type, new_shape, 0), ov::Exception); }
147+
}
148+
149+
TEST_F(FunctionalOffloadTensorTest, read_too_big_offset) {
150+
{
151+
// offset + data_size > file_size
152+
EXPECT_THROW(std::ignore = read_tensor_data(file_name, ov_type, shape, 1), ov::Exception);
153+
// offset == file_size
154+
EXPECT_THROW(std::ignore = read_tensor_data(file_name, ov_type, shape, data_size), ov::Exception);
155+
// offset > file_size
156+
EXPECT_THROW(std::ignore = read_tensor_data(file_name, ov_type, shape, data_size + 1), ov::Exception);
157+
}
158+
}
159+
160+
TEST_F(FunctionalOffloadTensorTest, read_dynamic_shape) {
161+
{
162+
ov::Tensor tensor;
163+
EXPECT_NO_THROW(tensor = read_tensor_data(file_name, ov_type, PartialShape::dynamic(1), 0));
164+
EXPECT_EQ(0, memcmp(tensor.data(), init_values.data(), data_size));
165+
}
166+
{
167+
ov::Tensor tensor;
168+
EXPECT_NO_THROW(tensor = read_tensor_data(file_name));
169+
EXPECT_EQ(0, memcmp(tensor.data(), init_values.data(), data_size));
170+
}
171+
}
172+
173+
TEST_F(FunctionalOffloadTensorTest, read_1_dynamic_dimension) {
174+
{
175+
ov::Tensor tensor;
176+
auto shape_with_1_dynamic_dimension = shape;
177+
size_t dynamic_dimension_number = shape_with_1_dynamic_dimension.size() - 1;
178+
shape_with_1_dynamic_dimension[dynamic_dimension_number] = -1;
179+
EXPECT_NO_THROW(tensor = read_tensor_data(file_name, ov_type, shape_with_1_dynamic_dimension, 0));
180+
EXPECT_EQ(tensor.get_shape()[dynamic_dimension_number], shape.get_shape()[dynamic_dimension_number]);
181+
EXPECT_EQ(0, memcmp(tensor.data(), init_values.data(), data_size));
182+
}
183+
}
184+
185+
TEST_F(FunctionalOffloadTensorTest, read_wrong_dynamic_shape) {
186+
{
187+
auto shape_with_1_dynamic_dimension = shape;
188+
shape_with_1_dynamic_dimension[shape_with_1_dynamic_dimension.size() - 1] = -1;
189+
shape_with_1_dynamic_dimension[shape_with_1_dynamic_dimension.size() - 2] = 100;
190+
EXPECT_THROW(std::ignore = read_tensor_data(file_name, ov_type, shape_with_1_dynamic_dimension, 0),
191+
ov::Exception);
192+
}
193+
}
194+
195+
TEST_F(FunctionalOffloadTensorTest, read_type_doesnt_fit_file_size) {
196+
{
197+
std::ofstream fout(file_name, std::ios::binary);
198+
fout.write(reinterpret_cast<char*>(init_values.data()), data_size - 1);
199+
}
200+
ASSERT_TRUE(std::filesystem::exists(file_name));
201+
202+
{ EXPECT_THROW(std::ignore = read_tensor_data(file_name, ov::element::f32), ov::Exception); }
203+
}
204+
} // namespace ov::test

0 commit comments

Comments
 (0)