Skip to content

Commit

Permalink
[fix][test] Fixup binary index unit test error
Browse files Browse the repository at this point in the history
  • Loading branch information
@LiuRuoyu01 @rock-git
LiuRuoyu01 authored and rock-git committed Jan 2, 2025
1 parent f1a5ffd commit 03441be
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 61 deletions.
18 changes: 16 additions & 2 deletions test/unit_test/vector/test_vector_index_binary_flat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ TEST_F(VectorIndexBinaryFlatTest, Create) {
pb::common::VectorIndexParameter index_parameter;
index_parameter.set_vector_index_type(::dingodb::pb::common::VectorIndexType::VECTOR_INDEX_TYPE_BINARY_FLAT);
index_parameter.mutable_binary_flat_parameter()->set_dimension(dimension);
index_parameter.mutable_binary_flat_parameter()->set_metric_type(::dingodb::pb::common::MetricType::METRIC_TYPE_NONE);
index_parameter.mutable_binary_flat_parameter()->set_metric_type(
::dingodb::pb::common::MetricType::METRIC_TYPE_NONE);
vector_index_binary_flat =
VectorIndexFactory::NewBinaryFlat(id, index_parameter, kEpoch, kRange, vector_index_thread_pool);
EXPECT_EQ(vector_index_binary_flat.get(), nullptr);
Expand All @@ -113,7 +114,8 @@ TEST_F(VectorIndexBinaryFlatTest, Create) {
pb::common::VectorIndexParameter index_parameter;
index_parameter.set_vector_index_type(::dingodb::pb::common::VectorIndexType::VECTOR_INDEX_TYPE_BINARY_FLAT);
index_parameter.mutable_binary_flat_parameter()->set_dimension(dimension);
index_parameter.mutable_binary_flat_parameter()->set_metric_type(::dingodb::pb::common::MetricType::METRIC_TYPE_HAMMING);
index_parameter.mutable_binary_flat_parameter()->set_metric_type(
::dingodb::pb::common::MetricType::METRIC_TYPE_HAMMING);
vector_index_binary_flat =
VectorIndexFactory::NewBinaryFlat(id, index_parameter, kEpoch, kRange, vector_index_thread_pool);
EXPECT_NE(vector_index_binary_flat.get(), nullptr);
Expand Down Expand Up @@ -403,6 +405,8 @@ TEST_F(VectorIndexBinaryFlatTest, Add) {
std::string byte_to_str(1, static_cast<char>(value));
vector_with_id.mutable_vector()->add_binary_values(byte_to_str);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);

Expand All @@ -423,6 +427,8 @@ TEST_F(VectorIndexBinaryFlatTest, Add) {
std::string byte_to_str(1, static_cast<char>(value));
vector_with_id.mutable_vector()->add_binary_values(byte_to_str);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand Down Expand Up @@ -530,6 +536,8 @@ TEST_F(VectorIndexBinaryFlatTest, UpsertWithDuplicated) {
std::string byte_to_str(1, static_cast<char>(value));
vector_with_id.mutable_vector()->add_binary_values(byte_to_str);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand All @@ -543,6 +551,8 @@ TEST_F(VectorIndexBinaryFlatTest, UpsertWithDuplicated) {
std::string byte_to_str(1, static_cast<char>(value));
vector_with_id.mutable_vector()->add_binary_values(byte_to_str);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand Down Expand Up @@ -594,6 +604,8 @@ TEST_F(VectorIndexBinaryFlatTest, Upsert) {
std::string byte_to_str(1, static_cast<char>(value));
vector_with_id.mutable_vector()->add_binary_values(byte_to_str);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand Down Expand Up @@ -659,6 +671,8 @@ TEST_F(VectorIndexBinaryFlatTest, Upsert) {
std::string byte_to_str(1, static_cast<char>(value));
vector_with_id.mutable_vector()->add_binary_values(byte_to_str);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand Down
74 changes: 15 additions & 59 deletions test/unit_test/vector/test_vector_index_binary_ivf_flat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ TEST_F(VectorIndexBinaryIvfFlatTest, Create) {
pb::common::VectorIndexParameter index_parameter;
index_parameter.set_vector_index_type(::dingodb::pb::common::VectorIndexType::VECTOR_INDEX_TYPE_BINARY_IVF_FLAT);
index_parameter.mutable_binary_ivf_flat_parameter()->set_dimension(64);
index_parameter.mutable_binary_ivf_flat_parameter()->set_metric_type(::dingodb::pb::common::MetricType::METRIC_TYPE_NONE);
index_parameter.mutable_binary_ivf_flat_parameter()->set_metric_type(
::dingodb::pb::common::MetricType::METRIC_TYPE_NONE);
vector_index_binary_ivf_flat =
VectorIndexFactory::NewBinaryIVFFlat(id, index_parameter, k_epoch, kRange, vector_index_thread_pool);
EXPECT_EQ(vector_index_binary_ivf_flat.get(), nullptr);
Expand All @@ -148,7 +149,7 @@ TEST_F(VectorIndexBinaryIvfFlatTest, Create) {
index_parameter.mutable_binary_ivf_flat_parameter()->set_dimension(dimension);
index_parameter.mutable_binary_ivf_flat_parameter()->set_metric_type(
::dingodb::pb::common::MetricType::METRIC_TYPE_HAMMING);
index_parameter.mutable_binary_ivf_flat_parameter()->set_ncentroids(0);
index_parameter.mutable_binary_ivf_flat_parameter()->set_ncentroids(0);
vector_index_binary_ivf_flat =
VectorIndexFactory::NewBinaryIVFFlat(id, index_parameter, k_epoch, kRange, vector_index_thread_pool);
EXPECT_NE(vector_index_binary_ivf_flat.get(), nullptr);
Expand Down Expand Up @@ -226,18 +227,6 @@ TEST_F(VectorIndexBinaryIvfFlatTest, SearchNotTrain) {
for (int j = 0; j < dim; j++) data_base[dim * i + j] = distrib(rng);
data_base[dim * i] += i / 255;
}

for (size_t i = 0; i < data_base_size; i++) {
LOG(INFO) << "[" << i << "]" << " [";
for (faiss::idx_t j = 0; j < dim; j++) {
if (0 != j) {
LOG(INFO) << ",";
}
LOG(INFO) << std::setw(10) << data_base[i * dim + j];
}

LOG(INFO) << "]";
}
}

// ok
Expand Down Expand Up @@ -275,18 +264,6 @@ TEST_F(VectorIndexBinaryIvfFlatTest, RangeSearchNotTrain) {
for (int j = 0; j < dim; j++) data_base[dim * i + j] = distrib(rng);
data_base[dim * i] += i / 255;
}

for (size_t i = 0; i < data_base_size; i++) {
LOG(INFO) << "[" << i << "]" << " [";
for (faiss::idx_t j = 0; j < dim; j++) {
if (0 != j) {
LOG(INFO) << ",";
}
LOG(INFO) << std::setw(10) << data_base[i * dim + j];
}

LOG(INFO) << "]";
}
}

// ok
Expand Down Expand Up @@ -324,6 +301,8 @@ TEST_F(VectorIndexBinaryIvfFlatTest, AddNotTrain) {
std::string byte_data(1, static_cast<char>(data_base[i]));
vector_with_id.mutable_vector()->add_binary_values(byte_data);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);

Expand Down Expand Up @@ -362,18 +341,6 @@ TEST_F(VectorIndexBinaryIvfFlatTest, TrainVectorWithId) {
for (int j = 0; j < dim; j++) data_base[dim * i + j] = distrib(rng);
data_base[dim * i] += i / 255;
}

for (size_t i = 0; i < data_base_size; i++) {
LOG(INFO) << "[" << i << "]" << " [";
for (faiss::idx_t j = 0; j < dim; j++) {
if (0 != j) {
LOG(INFO) << ",";
}
LOG(INFO) << std::setw(10) << data_base[i * dim + j];
}

LOG(INFO) << "]";
}
}

// invalid. no data
Expand All @@ -398,9 +365,9 @@ TEST_F(VectorIndexBinaryIvfFlatTest, TrainVectorWithId) {
vector_with_ids.push_back(vector_with_id);
}

//not align
// not align
pb::common::VectorWithId vector_with_id;
for (int j = 0; j < dim-1; j++) {
for (int j = 0; j < dim - 1; j++) {
std::string byte_data(1, static_cast<char>(data_base_not_align[(data_base_size - 1) * dim + j]));
vector_with_id.mutable_vector()->add_binary_values(byte_data);
}
Expand Down Expand Up @@ -470,18 +437,6 @@ TEST_F(VectorIndexBinaryIvfFlatTest, Train) {
for (int j = 0; j < dim; j++) data_base[dim * i + j] = distrib(rng);
data_base[dim * i] += i / 255;
}

for (size_t i = 0; i < data_base_size; i++) {
LOG(INFO) << "[" << i << "]" << " [";
for (faiss::idx_t j = 0; j < dim; j++) {
if (0 != j) {
LOG(INFO) << ",";
}
LOG(INFO) << std::setw(10) << data_base[i * dim + j];
}

LOG(INFO) << "]";
}
}

// invalid. no data
Expand Down Expand Up @@ -567,6 +522,8 @@ TEST_F(VectorIndexBinaryIvfFlatTest, Add) {
std::string byte_data(1, static_cast<char>(data_base[i]));
vector_with_id.mutable_vector()->add_binary_values(byte_data);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);

Expand All @@ -586,6 +543,8 @@ TEST_F(VectorIndexBinaryIvfFlatTest, Add) {
std::string byte_data(1, static_cast<char>(data_base[id * dim + i]));
vector_with_id.mutable_vector()->add_binary_values(byte_data);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand Down Expand Up @@ -648,6 +607,8 @@ TEST_F(VectorIndexBinaryIvfFlatTest, Upsert) {
std::string byte_data(1, static_cast<char>(data_base[id * dim + i]));
vector_with_id.mutable_vector()->add_binary_values(byte_data);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand Down Expand Up @@ -675,6 +636,8 @@ TEST_F(VectorIndexBinaryIvfFlatTest, Upsert) {
std::string byte_data(1, static_cast<char>(data_base[id * dim + i]));
vector_with_id.mutable_vector()->add_binary_values(byte_data);
}
vector_with_id.mutable_vector()->set_dimension(dimension);
vector_with_id.mutable_vector()->set_value_type(::dingodb::pb::common::ValueType::UINT8);

vector_with_ids.push_back(vector_with_id);
}
Expand Down Expand Up @@ -1008,12 +971,10 @@ TEST_F(VectorIndexBinaryIvfFlatTest, SearchAfterLoad) {
ok = vector_index_binary_ivf_flat->Search(vector_with_ids, topk, {}, false, {}, results);
EXPECT_EQ(ok.error_code(), pb::error::Errno::OK);


for (const auto& result : results) {
DINGO_LOG(INFO) << "results:" << result.DebugString();
DINGO_LOG(INFO) << "";
}

}

// ok with param
Expand Down Expand Up @@ -1070,7 +1031,6 @@ TEST_F(VectorIndexBinaryIvfFlatTest, SearchAfterLoad) {
if (is_all_in_vector) {
DINGO_LOG(INFO) << "All Id in vectors ";
}

}
}

Expand All @@ -1086,7 +1046,6 @@ TEST_F(VectorIndexBinaryIvfFlatTest, RangeSearchAfterLoad) {
std::vector<pb::index::VectorWithDistanceResult> results;
ok = vector_index_binary_ivf_flat->RangeSearch(vector_with_ids, radius, {}, false, {}, results);
EXPECT_EQ(ok.error_code(), pb::error::Errno::EVECTOR_INVALID);

}

// invalid param failed, topk == 0, return OK
Expand All @@ -1106,7 +1065,6 @@ TEST_F(VectorIndexBinaryIvfFlatTest, RangeSearchAfterLoad) {

ok = vector_index_binary_ivf_flat->RangeSearch(vector_with_ids, radius, {}, false, {}, results);
EXPECT_EQ(ok.error_code(), pb::error::Errno::OK);

}

// ok
Expand All @@ -1127,12 +1085,10 @@ TEST_F(VectorIndexBinaryIvfFlatTest, RangeSearchAfterLoad) {
ok = vector_index_binary_ivf_flat->RangeSearch(vector_with_ids, radius, {}, false, {}, results);
EXPECT_EQ(ok.error_code(), pb::error::Errno::OK);


for (const auto& result : results) {
DINGO_LOG(INFO) << "results:" << result.DebugString();
DINGO_LOG(INFO) << "";
}

}

// ok with param
Expand Down

0 comments on commit 03441be

Please sign in to comment.