diff --git a/src/DescriptorsCommand.cc b/src/DescriptorsCommand.cc index fea90303..f674616e 100644 --- a/src/DescriptorsCommand.cc +++ b/src/DescriptorsCommand.cc @@ -201,12 +201,20 @@ Json::Value FindDescriptorSet::construct_responses( AddDescriptorSet::AddDescriptorSet() : DescriptorsCommand("AddDescriptorSet") { _storage_sets = VDMSConfig::instance()->get_path_descriptors(); - _flinng_num_rows = 3; // set based on the default values of Flinng - _flinng_cells_per_row = 1000; - _flinng_num_hash_tables = 10; - _flinng_hashes_per_table = 12; - _flinng_sub_hash_bits = 2; - _flinng_cut_off = 6; + + //Flinng params + _flinng_num_rows = static_cast(VDMSConfig::instance()->get_flinng_num_rows().value_or(3)); + _flinng_cells_per_row = static_cast(VDMSConfig::instance()->get_flinng_cells_per_row().value_or(1000)); + _flinng_num_hash_tables = static_cast(VDMSConfig::instance()->get_flinng_num_hash_tables().value_or(10)); + _flinng_hashes_per_table = static_cast(VDMSConfig::instance()->get_flinng_hashes_per_table().value_or(12)); + + //IVF params + _ivf_nlist = static_cast(VDMSConfig::instance()->get_ivf_nlist().value_or(16)); + + //HNSW params + _hnsw_efsearch = static_cast(VDMSConfig::instance()->get_hnsw_efsearch().value_or(64)); + _hnsw_efConstruction = static_cast(VDMSConfig::instance()->get_hnsw_efConstruction().value_or(96)); + _hnsw_M = static_cast(VDMSConfig::instance()->get_hnsw_M().value_or(48)); //_use_aws_storage = VDMSConfig::instance()->get_aws_flag(); } @@ -228,6 +236,7 @@ int AddDescriptorSet::construct_protobuf(PMGDQuery &query, props[VDMS_DESC_SET_DIM_PROP] = cmd["dimensions"].asInt(); props[VDMS_DESC_SET_PATH_PROP] = desc_set_path; props[VDMS_DESC_SET_ENGIN_PROP] = cmd["engine"].asString(); + if (props[VDMS_DESC_SET_ENGIN_PROP] == "Flinng") { if (cmd.isMember("flinng_num_rows")) _flinng_num_rows = cmd["flinng_num_rows"].asInt(); @@ -242,7 +251,21 @@ int AddDescriptorSet::construct_protobuf(PMGDQuery &query, if (cmd.isMember("flinng_cut_off")) _flinng_cut_off = cmd["flinng_cut_off"].asInt(); } - + + if (props[VDMS_DESC_SET_ENGIN_PROP] == "FaissIVFFlat"){ + if (cmd.isMember("ivf_nlist")) + _ivf_nlist = cmd["ivf_nlist"].asInt(); + } + + if (props[VDMS_DESC_SET_ENGIN_PROP] == "FaissHNSWFlat"){ + if (cmd.isMember("hnsw_efsearch")) + _hnsw_efsearch = cmd["hnsw_efsearch"].asInt(); + if (cmd.isMember("hnsw_efConstruction")) + _hnsw_efConstruction = cmd["_hnsw_efConstruction"].asInt(); + if (cmd.isMember("hnsw_M")) + _hnsw_M = cmd["hnsw_M"].asInt(); + } + Json::Value constraints; constraints[VDMS_DESC_SET_NAME_PROP].append("=="); constraints[VDMS_DESC_SET_NAME_PROP].append(cmd["name"].asString()); @@ -312,10 +335,15 @@ Json::Value AddDescriptorSet::construct_responses( // We can probably set up a mechanism // to fix a broken link when detected later, same with images. VCL::DescriptorParams *param = nullptr; + try { param = new VCL::DescriptorParams(_flinng_num_rows, _flinng_cells_per_row, _flinng_num_hash_tables, - _flinng_hashes_per_table); + _flinng_hashes_per_table, + _ivf_nlist, + _hnsw_efsearch, + _hnsw_efConstruction, + _hnsw_M); VCL::DescriptorSet desc_set(desc_set_path, dimensions, _eng, metric, param); if (_use_aws_storage) { diff --git a/src/DescriptorsCommand.h b/src/DescriptorsCommand.h index e872cd6e..66dcdb4d 100644 --- a/src/DescriptorsCommand.h +++ b/src/DescriptorsCommand.h @@ -109,8 +109,12 @@ class AddDescriptorSet : public DescriptorsCommand { _flinng_sub_hash_bits; // sub_hash_bits * hashes_per_table must be // less than 32, otherwise segfault will happen uint64_t _flinng_cut_off; - // bool _use_aws_storage; + uint64_t _ivf_nlist; //Nlist for IVF Index + uint64_t _hnsw_efsearch; //Efsearch for the search width of hnsw + uint64_t _hnsw_efConstruction; //Efconstruction for the width of hnsw build + uint64_t _hnsw_M; //typically Efconstruction=2*M +// bool _use_aws_storage; public: AddDescriptorSet(); diff --git a/src/VDMSConfig.cc b/src/VDMSConfig.cc index c87ecd98..3f064852 100644 --- a/src/VDMSConfig.cc +++ b/src/VDMSConfig.cc @@ -112,6 +112,16 @@ VDMSConfig::VDMSConfig(std::string config_file) { proxy_port = std::nullopt; proxy_scheme = std::nullopt; + flinng_num_rows = std::nullopt; + flinng_cells_per_row = std::nullopt; + flinng_num_hash_tables = std::nullopt; + flinng_hashes_per_table = std::nullopt; + ivf_nlist = std::nullopt; + hnsw_efsearch = std::nullopt; + hnsw_efConstruction = std::nullopt; + hnsw_M = std::nullopt; + + bool parsingSuccessful = reader.parse(file, json_config); if (!parsingSuccessful) { @@ -364,6 +374,74 @@ void VDMSConfig::build_dirs() { if (aws_log_level_map.find(aws_log_level_value) != aws_log_level_map.end()) { aws_log_level = aws_log_level_map.at(aws_log_level_value); } + +//Descriptor parameters + + // flinng_num_rows + if (exists_key(PARAM_FLINNG_NUM_ROWS)) { + value = get_string_value(PARAM_FLINNG_NUM_ROWS, KEY_NOT_FOUND); + flinng_num_rows = std::optional{stoi(value)}; + } else { + flinng_num_rows = std::optional{3}; + } + + // flinng_cells_per_row + if (exists_key(PARAM_FLINNG_CELLS_PER_ROW)) { + value = get_string_value(PARAM_FLINNG_CELLS_PER_ROW, KEY_NOT_FOUND); + flinng_cells_per_row = std::optional{stoi(value)}; + } else { + flinng_cells_per_row = std::optional{1000}; + } + + // flinng_num_hash_tables + if (exists_key(PARAM_FLINNG_NUM_HASH_TABLES)) { + value = get_string_value(PARAM_FLINNG_NUM_HASH_TABLES, KEY_NOT_FOUND); + flinng_num_hash_tables = std::optional{stoi(value)}; + } else { + flinng_num_hash_tables = std::optional{10}; + } + + // flinng_hashes_per_table + if (exists_key(PARAM_FLINNG_HASHES_PER_TABLE)) { + value = get_string_value(PARAM_FLINNG_HASHES_PER_TABLE, KEY_NOT_FOUND); + flinng_hashes_per_table = std::optional{stoi(value)}; + } else { + flinng_hashes_per_table = std::optional{12}; + } + + // ivf_nlist + if (exists_key(PARAM_IVF_NLIST)) { + value = get_string_value(PARAM_IVF_NLIST, KEY_NOT_FOUND); + ivf_nlist = std::optional{stoi(value)}; + } else { + ivf_nlist = std::optional{16}; + } + +// hnsw_efsearch + if (exists_key(PARAM_HNSW_EFSEARCH)) { + value = get_string_value(PARAM_HNSW_EFSEARCH, KEY_NOT_FOUND); + hnsw_efsearch = std::optional{stoi(value)}; + } else { + hnsw_efsearch = std::optional{64}; + } + + // hnsw_efConstruction + if (exists_key(PARAM_HNSW_EFCONSTRUCTION)) { + value = get_string_value(PARAM_HNSW_EFCONSTRUCTION, KEY_NOT_FOUND); + hnsw_efConstruction = std::optional{stoi(value)}; + } else { + hnsw_efConstruction = std::optional{96}; + } + + // hnsw_M + if (exists_key(PARAM_HNSW_M)) { + value = get_string_value(PARAM_HNSW_M, KEY_NOT_FOUND); + hnsw_M = std::optional{stoi(value)}; + } else { + hnsw_M = std::optional{48}; + } + + } bool VDMSConfig::exists_key(const std::string &key) { diff --git a/src/VDMSConfig.h b/src/VDMSConfig.h index 298e3301..ee8e5e78 100644 --- a/src/VDMSConfig.h +++ b/src/VDMSConfig.h @@ -90,6 +90,15 @@ const std::string PARAM_PROXY_SCHEME = "proxy_scheme"; const std::string PARAM_USE_ENDPOINT = "use_endpoint"; const std::string PARAM_AWS_LOG_LEVEL = "aws_log_level"; +const std::string PARAM_FLINNG_NUM_ROWS = "flinng_num_rows"; +const std::string PARAM_FLINNG_CELLS_PER_ROW = "flinng_cells_per_row"; +const std::string PARAM_FLINNG_NUM_HASH_TABLES = "flinng_num_hash_tables"; +const std::string PARAM_FLINNG_HASHES_PER_TABLE = "flinng_hashes_per_table"; +const std::string PARAM_IVF_NLIST = "ivf_nlist"; +const std::string PARAM_HNSW_EFSEARCH = "hnsw_efsearch"; +const std::string PARAM_HNSW_EFCONSTRUCTION = "hnsw_efConstruction"; +const std::string PARAM_HNSW_M = "hnsw_M"; + namespace VDMS { class VDMSConfig { @@ -139,6 +148,18 @@ class VDMSConfig { return aws_log_level; } + //Descriptor Optional Parameters + const std::optional &get_flinng_num_rows() { return flinng_num_rows; } + const std::optional &get_flinng_cells_per_row() { return flinng_cells_per_row; } + const std::optional &get_flinng_num_hash_tables() { return flinng_num_hash_tables; } + const std::optional &get_flinng_hashes_per_table() { return flinng_hashes_per_table; } + const std::optional &get_ivf_nlist() { return ivf_nlist; } + const std::optional &get_hnsw_efsearch() { return hnsw_efsearch; } + const std::optional &get_hnsw_efConstruction() { return hnsw_efConstruction; } + const std::optional &get_hnsw_M() { return hnsw_M; } + + + protected: static VDMSConfig *cfg; static std::mutex _mutex; @@ -172,6 +193,16 @@ class VDMSConfig { std::optional proxy_scheme; Aws::Utils::Logging::LogLevel aws_log_level; + std::optional flinng_num_rows; + std::optional flinng_cells_per_row; + std::optional flinng_num_hash_tables; + std::optional flinng_hashes_per_table; + std::optional ivf_nlist; + std::optional hnsw_efsearch; + std::optional hnsw_efConstruction; + std::optional hnsw_M; + + void expand_directory_layer( std::vector *> *p_directory_list, int current_layer); @@ -193,6 +224,16 @@ class VDMSConfig { proxy_host = std::nullopt; proxy_port = std::nullopt; proxy_scheme = std::nullopt; + + flinng_num_rows = std::optional{3}; + flinng_cells_per_row = std::optional{1000}; + flinng_num_hash_tables = std::optional{10}; + flinng_hashes_per_table = std::optional{12}; + ivf_nlist = std::optional{16}; + hnsw_efsearch = std::optional{64}; + hnsw_efConstruction = std::optional{96}; + hnsw_M = std::optional{48}; + } }; diff --git a/src/vcl/DescriptorParams.cc b/src/vcl/DescriptorParams.cc index 4e36bf4b..7be5b377 100644 --- a/src/vcl/DescriptorParams.cc +++ b/src/vcl/DescriptorParams.cc @@ -36,7 +36,11 @@ DescriptorParams::DescriptorParams(uint64_t numrows = 3, uint64_t numhashtables = (1 << 9), uint64_t hashespertable = 14, uint64_t subhashbits = 2, - uint64_t cutoff = 6) { + uint64_t cutoff = 6, + uint64_t ivf_nlist = 16, + uint64_t hnsw_efsearch = 64, + uint64_t hnsw_efConstruction = 96, + uint64_t hnsw_M = 48) { this->num_rows = numrows; this->cells_per_row = cellsperrow; this->num_hash_tables = numhashtables; @@ -45,4 +49,10 @@ DescriptorParams::DescriptorParams(uint64_t numrows = 3, subhashbits; // sub_hash_bits * hashes_per_table must be less than 32, // otherwise segfault will happen this->cut_off = cutoff; + //IVF Parameters + this->ivf_nlist = ivf_nlist; + //HNSW Parameters + this->hnsw_efsearch = hnsw_efsearch; + this->hnsw_efConstruction = hnsw_efConstruction; + this->hnsw_M = hnsw_M; } diff --git a/src/vcl/DescriptorParams.h b/src/vcl/DescriptorParams.h index c54ba2d7..5a5b64a5 100644 --- a/src/vcl/DescriptorParams.h +++ b/src/vcl/DescriptorParams.h @@ -62,16 +62,33 @@ class DescriptorParams { // 32, otherwise segfault will happen uint64_t cut_off; + /*Params needed for IVF*/ + uint64_t ivf_nlist; + + /*Params needed for HNSW*/ + uint64_t hnsw_efsearch; + uint64_t hnsw_efConstruction; + uint64_t hnsw_M; + + DescriptorParams(uint64_t numrows = 3, uint64_t cellsperrow = (1 << 12), uint64_t numhashtables = (1 << 9), uint64_t hashespertable = 14, uint64_t subhashbits = 2, - uint64_t cutoff = 6) { + uint64_t cutoff = 6, + uint64_t ivf_nlist = 16, + uint64_t hnsw_efsearch = 64, + uint64_t hnsw_efConstruction = 96, + uint64_t hnsw_M = 48) { this->num_rows = numrows; this->cells_per_row = cellsperrow; this->num_hash_tables = numhashtables; this->hashes_per_table = hashespertable; this->sub_hash_bits = subhashbits; this->cut_off = cutoff; + this->ivf_nlist = ivf_nlist; + this->hnsw_efsearch = hnsw_efsearch; + this->hnsw_efConstruction = hnsw_efConstruction; + this->hnsw_M = hnsw_M; } }; }; // namespace VCL diff --git a/src/vcl/DescriptorSet.cc b/src/vcl/DescriptorSet.cc index e8f5cadd..d201c78b 100644 --- a/src/vcl/DescriptorSet.cc +++ b/src/vcl/DescriptorSet.cc @@ -80,7 +80,7 @@ DescriptorSet::DescriptorSet(const std::string &set_path, unsigned dim, if (eng == DescriptorSetEngine(FaissFlat)) _set = new FaissFlatDescriptorSet(set_path, dim, metric); else if (eng == DescriptorSetEngine(FaissIVFFlat)) - _set = new FaissIVFFlatDescriptorSet(set_path, dim, metric); + _set = new FaissIVFFlatDescriptorSet(set_path, dim, metric, param); else if (eng == DescriptorSetEngine(TileDBDense)) _set = new TDBDenseDescriptorSet(set_path, dim, metric); else if (eng == DescriptorSetEngine(TileDBSparse)) @@ -88,7 +88,7 @@ DescriptorSet::DescriptorSet(const std::string &set_path, unsigned dim, else if (eng == DescriptorSetEngine(Flinng)) _set = new FlinngDescriptorSet(set_path, dim, metric, param); else if (eng == DescriptorSetEngine(FaissHNSWFlat)) - _set = new FaissHNSWFlatDescriptorSet(set_path, dim, metric); + _set = new FaissHNSWFlatDescriptorSet(set_path, dim, metric, param); else { std::cerr << "Index Not supported" << std::endl; throw VCLException(UnsupportedIndex, "Index not supported"); diff --git a/src/vcl/FaissDescriptorSet.cc b/src/vcl/FaissDescriptorSet.cc index 4adb5e52..fb826b77 100644 --- a/src/vcl/FaissDescriptorSet.cc +++ b/src/vcl/FaissDescriptorSet.cc @@ -284,7 +284,7 @@ FaissIVFFlatDescriptorSet::FaissIVFFlatDescriptorSet( } FaissIVFFlatDescriptorSet::FaissIVFFlatDescriptorSet( - const std::string &set_path, unsigned dim, DistanceMetric metric) + const std::string &set_path, unsigned dim, DistanceMetric metric, VCL::DescriptorParams *par) : FaissDescriptorSet(set_path, dim) { // TODO: Revise nlist param for future optimizations. // 4 is a suggested value by faiss for the IVFFlat index, @@ -292,8 +292,10 @@ FaissIVFFlatDescriptorSet::FaissIVFFlatDescriptorSet( // int nlist = 4; // default value of 4 is too low for any sizeable dataset + + // int nlist = 16 - int nlist = 16; + int nlist = par->ivf_nlist; //nlist is a configurable parameter if (metric == L2) { faiss::IndexFlatL2 *quantizer = new faiss::IndexFlatL2(_dimensions); @@ -384,13 +386,17 @@ FaissHNSWFlatDescriptorSet::FaissHNSWFlatDescriptorSet( } FaissHNSWFlatDescriptorSet::FaissHNSWFlatDescriptorSet( - const std::string &set_path, unsigned dim, DistanceMetric metric) + const std::string &set_path, unsigned dim, DistanceMetric metric,VCL::DescriptorParams *par) : FaissDescriptorSet(set_path, dim) { - int hnsw_M = 48; + int hnsw_M = par->hnsw_M ; + int hnsw_efConstruction= par->hnsw_efConstruction; + int hnsw_efsearch = par->hnsw_efsearch; + if (metric == L2) { _index = new faiss::IndexHNSWFlat(dim, hnsw_M, faiss::METRIC_L2); - ((faiss::IndexHNSWFlat *)_index)->hnsw.efConstruction = 96; + ((faiss::IndexHNSWFlat *)_index)->hnsw.efConstruction = hnsw_efConstruction; + ((faiss::IndexHNSWFlat *)_index)->hnsw.efSearch = hnsw_efsearch; } else { // only metric L2 is supported for HNSWFLAT for FAISS v1.7.4 // newer version of Faiss e.g. V1.8.0 supports I.P. metric for HNSW @@ -401,14 +407,15 @@ FaissHNSWFlatDescriptorSet::FaissHNSWFlatDescriptorSet( void FaissHNSWFlatDescriptorSet::search(float *query, unsigned n_queries, unsigned k, long *descriptors, float *distances) { - ((faiss::IndexHNSWFlat *)_index)->hnsw.efSearch = 64; + //((faiss::IndexHNSWFlat *)_index)->hnsw.efSearch = 64; + + //efSearch is modified to be set as a parameter when the index is constucted + // set according to // https://github.com/facebookresearch/faiss/wiki/Indexing-1M-vectors for R@1 // accuracy of 0.9779 // The higher the value the slower the search is but better accuracy // efsearch is a runtime parameter. - // ToDO - VDMS should expose an API to set runtime parameters to users of the - // different indices _index->search(n_queries, query, k, distances, descriptors); } diff --git a/src/vcl/FaissDescriptorSet.h b/src/vcl/FaissDescriptorSet.h index d484969e..0a6f7611 100644 --- a/src/vcl/FaissDescriptorSet.h +++ b/src/vcl/FaissDescriptorSet.h @@ -105,8 +105,9 @@ class FaissIVFFlatDescriptorSet : public FaissDescriptorSet { public: FaissIVFFlatDescriptorSet(const std::string &set_path); - FaissIVFFlatDescriptorSet(const std::string &set_path, unsigned dim, - DistanceMetric metric); + // FaissIVFFlatDescriptorSet(const std::string &set_path, unsigned dim, DistanceMetric metric); + FaissIVFFlatDescriptorSet(const std::string &set_path, unsigned dim, + DistanceMetric metric, VCL::DescriptorParams *par); long add(float *descriptors, unsigned n_descriptors, long *classes); }; @@ -115,11 +116,13 @@ class FaissHNSWFlatDescriptorSet : public FaissDescriptorSet { public: FaissHNSWFlatDescriptorSet(const std::string &set_path); + // FaissHNSWFlatDescriptorSet(const std::string &set_path, unsigned dim,DistanceMetric metric); FaissHNSWFlatDescriptorSet(const std::string &set_path, unsigned dim, - DistanceMetric metric); + DistanceMetric metric, VCL::DescriptorParams *par); void search(float *query, unsigned n_queries, unsigned k, long *descriptors, float *distances); + }; }; // namespace VCL diff --git a/tests/unit_tests/config-for-vdms-config-v1-tests.json b/tests/unit_tests/config-for-vdms-config-v1-tests.json index bb4d8404..e78a50aa 100644 --- a/tests/unit_tests/config-for-vdms-config-v1-tests.json +++ b/tests/unit_tests/config-for-vdms-config-v1-tests.json @@ -11,5 +11,14 @@ "proxy_host": "a.proxy.to.be.tested.intel.com", "proxy_port": 8000, "proxy_scheme": "http", - "more-info": "github.com/IntelLabs/vdms" + "more-info": "github.com/IntelLabs/vdms", + "flinng_num_rows": 3, + "flinng_cells_per_row": 1000, + "flinng_num_hash_tables": 10, + "flinng_hashes_per_table": 12, + "ivf_nlist": 16, + "hnsw_efsearch": 64, + "hnsw_efConstruction": 96, + "hnsw_M":48 + }