-
Notifications
You must be signed in to change notification settings - Fork 83
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: cqy123456 <[email protected]>
- Loading branch information
Showing
7 changed files
with
282 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| // Copyright (C) 2019-2023 Zilliz. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance | ||
| // with the License. You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software distributed under the License | ||
| // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express | ||
| // or implied. See the License for the specific language governing permissions and limitations under the License | ||
|
|
||
| #include "io/trailer.h" | ||
|
|
||
| #include <cstring> | ||
| #include <fstream> | ||
|
|
||
| namespace { | ||
| static constexpr size_t kBlockSize = 4096; | ||
| uint32_t | ||
| CalculateCheckSum(const uint8_t* data, int64_t size) { | ||
| uint32_t checksum = 0; | ||
| for (auto i = 0; i < size; i++) { | ||
| checksum ^= data[i]; // xor | ||
| } | ||
| return checksum; | ||
| } | ||
|
|
||
| uint32_t | ||
| GetFilesCheckSum(std::vector<std::string> files) { | ||
| uint32_t checksum = 0; | ||
| auto buffer = std::shared_ptr<uint8_t[]>(new uint8_t[kBlockSize]); | ||
| for (auto& file_name : files) { | ||
| std::ifstream reader(file_name.c_str(), std::ios::binary); | ||
| if (!reader) { | ||
| LOG_KNOWHERE_WARNING_ << file_name << "not exist, skip calculate check sum for this file."; | ||
| continue; | ||
| } | ||
| while (reader.read((char*)buffer.get(), kBlockSize)) { | ||
| std::streamsize read_size = reader.gcount(); | ||
| checksum ^= CalculateCheckSum(buffer.get(), read_size); | ||
| } | ||
| } | ||
| return checksum; | ||
| } | ||
| } // namespace | ||
|
|
||
| namespace knowhere { | ||
| void | ||
| AddTrailerForMemoryIO(MemoryIOWriter& writer, const std::string& name, const Version& version) { | ||
| auto trailer_ptr = std::make_unique<Trailer>(); | ||
| auto size = writer.tellg(); | ||
| trailer_ptr->SetIndexBinarySize(size); | ||
| trailer_ptr->SetCheckSum(CalculateCheckSum(writer.data(), size)); | ||
| trailer_ptr->SetVersion(version.VersionNumber()); | ||
| trailer_ptr->SetIndexName(name); | ||
| writer.write(trailer_ptr->bytes, KNOWHERE_TRAILER_SIZE); | ||
| } | ||
|
|
||
| bool | ||
| CheckTrailerForMemoryIO(MemoryIOReader& reader, const std::string& name) { | ||
| uint64_t bin_size = TRAILER_OFFSET(reader.size()); | ||
| if (bin_size < 0) { | ||
| LOG_KNOWHERE_WARNING_ << "The binary is too small and assume no Trailer, pass Trailer check."; | ||
| return true; | ||
| } | ||
|
|
||
| auto trailer_ptr = std::make_unique<Trailer>(); | ||
| auto pre_rp = reader.tellg(); | ||
| reader.seekg(bin_size); | ||
| reader.read(trailer_ptr.get(), KNOWHERE_TRAILER_SIZE); | ||
| reader.seekg(pre_rp); | ||
|
|
||
| if (!trailer_ptr->TrailerValidCheck()) { | ||
| LOG_KNOWHERE_WARNING_ << "Trailer not exist in Binary."; | ||
| return true; | ||
| } | ||
|
|
||
| auto version = Version(trailer_ptr->GetVersion()); | ||
| if (!Version::VersionSupport(version)) { | ||
| LOG_KNOWHERE_ERROR_ << "Index version(" << version.VersionNumber() << ") is not supported, Trailer check fail."; | ||
| return false; | ||
| } | ||
|
|
||
| if (trailer_ptr->GetIndexName() != name) { | ||
| LOG_KNOWHERE_ERROR_ << "Index type or data type is not correct(" << name << ")."; | ||
| return false; | ||
| } | ||
|
|
||
| if (trailer_ptr->GetIndexBinarySize() != bin_size) { | ||
| LOG_KNOWHERE_ERROR_ << "The size of index binary is not correct."; | ||
| return false; | ||
| } | ||
|
|
||
| if (CalculateCheckSum(reader.data(), bin_size) != trailer_ptr->GetCheckSum()) { | ||
| LOG_KNOWHERE_ERROR_ << "Binary checksum check fail."; | ||
| return false; | ||
| } | ||
| LOG_KNOWHERE_INFO_ << "Index Trailer check succeed."; | ||
| return true; | ||
| } | ||
|
|
||
| void | ||
| AddTrailerForFiles(const std::vector<std::string>& files, const std::string& trailer_file, const std::string& name, | ||
| const Version& version) { | ||
| auto trailer_ptr = std::make_unique<Trailer>(); | ||
| trailer_ptr->SetCheckSum(GetFilesCheckSum(files)); | ||
| trailer_ptr->SetVersion(version.VersionNumber()); | ||
| trailer_ptr->SetIndexName(name); | ||
|
|
||
| std::ofstream writer(trailer_file.c_str(), std::ios::binary); | ||
| writer.write((char*)trailer_ptr->bytes, KNOWHERE_TRAILER_SIZE); | ||
| writer.close(); | ||
| } | ||
|
|
||
| bool | ||
| CheckTrailerForFiles(const std::vector<std::string>& files, const std::string& trailer_file, const std::string& name) { | ||
| std::ifstream reader(trailer_file.c_str(), std::ios::binary); | ||
| if (!reader) { | ||
| LOG_KNOWHERE_WARNING_ << "Trailer file not exist."; | ||
| return true; | ||
| } | ||
| reader.seekg(0, std::ios::end); | ||
| auto fsize = reader.tellg(); | ||
|
|
||
| reader.seekg(0, std::ios::beg); | ||
| if (fsize != KNOWHERE_TRAILER_SIZE) { | ||
| LOG_KNOWHERE_ERROR_ << "Trailer size (" << fsize << ")not correct."; | ||
| return false; | ||
| } | ||
| auto trailer_ptr = std::make_unique<Trailer>(); | ||
| reader.read((char*)trailer_ptr.get()->bytes, KNOWHERE_TRAILER_SIZE); | ||
| if (!trailer_ptr->TrailerValidCheck()) { | ||
| LOG_KNOWHERE_WARNING_ << "Trailer flag not right."; | ||
| return false; | ||
| } | ||
|
|
||
| auto version = Version(trailer_ptr->GetVersion()); | ||
| if (!Version::VersionSupport(version)) { | ||
| LOG_KNOWHERE_ERROR_ << "Index version(" << version.VersionNumber() << ") is not supported, Trailer check fail."; | ||
| return false; | ||
| } | ||
|
|
||
| if (trailer_ptr->GetIndexName() != name) { | ||
| LOG_KNOWHERE_ERROR_ << "Index type or data type is not correct(" << name << ")."; | ||
| return false; | ||
| } | ||
|
|
||
| if (GetFilesCheckSum(files) != trailer_ptr->GetCheckSum()) { | ||
| LOG_KNOWHERE_ERROR_ << "Files checksum check fail."; | ||
| return false; | ||
| } | ||
| LOG_KNOWHERE_INFO_ << "Index Trailer check succeed."; | ||
| return true; | ||
| } | ||
|
|
||
| } // namespace knowhere |
Oops, something went wrong.