diff --git a/tiledb/sm/filesystem/s3.cc b/tiledb/sm/filesystem/s3.cc index ba6fa5adcce..2ad9470db28 100644 --- a/tiledb/sm/filesystem/s3.cc +++ b/tiledb/sm/filesystem/s3.cc @@ -1775,9 +1775,13 @@ std::optional S3::multipart_upload_state( } Status S3::set_multipart_upload_state( - const URI& uri, const MultiPartUploadState& state) { - const Aws::Http::URI aws_uri(uri.c_str()); - const std::string uri_path(aws_uri.GetPath().c_str()); + const std::string& uri, MultiPartUploadState& state) { + Aws::Http::URI aws_uri(uri.c_str()); + std::string uri_path(aws_uri.GetPath().c_str()); + + state.bucket = aws_uri.GetAuthority(); + state.key = aws_uri.GetPath(); + UniqueWriteLock unique_wl(&multipart_upload_rwlock_); multipart_upload_states_[uri_path] = state; diff --git a/tiledb/sm/filesystem/s3.h b/tiledb/sm/filesystem/s3.h index 8e00be54e6c..4e54d314465 100644 --- a/tiledb/sm/filesystem/s3.h +++ b/tiledb/sm/filesystem/s3.h @@ -378,7 +378,7 @@ class S3 { * @return Status */ Status set_multipart_upload_state( - const URI& uri, const S3::MultiPartUploadState& state); + const std::string& uri, S3::MultiPartUploadState& state); private: /* ********************************* */ diff --git a/tiledb/sm/filesystem/vfs.cc b/tiledb/sm/filesystem/vfs.cc index 94426538d97..d7fcf5bc78c 100644 --- a/tiledb/sm/filesystem/vfs.cc +++ b/tiledb/sm/filesystem/vfs.cc @@ -1827,7 +1827,9 @@ Status VFS::write(const URI& uri, const void* buffer, uint64_t buffer_size) { std::pair> VFS::multipart_upload_state(const URI& uri) { - if (uri.is_s3()) { + if (uri.is_file()) { + return {Status::Ok(), {}}; + } else if (uri.is_s3()) { #ifdef HAVE_S3 VFS::MultiPartUploadState state; auto s3_state = s3_.multipart_upload_state(uri); @@ -1836,8 +1838,6 @@ VFS::multipart_upload_state(const URI& uri) { } state.upload_id = s3_state->upload_id; state.part_number = s3_state->part_number; - state.bucket = s3_state->bucket; - state.s3_key = s3_state->key; state.status = s3_state->st; auto& completed_parts = s3_state->completed_parts; for (auto& entry : completed_parts) { @@ -1868,26 +1868,28 @@ VFS::multipart_upload_state(const URI& uri) { #endif } - return {Status::Ok(), nullopt}; + return {LOG_STATUS( + Status_VFSError("Unsupported URI schemes: " + uri.to_string())), + nullopt}; } Status VFS::set_multipart_upload_state( const URI& uri, const MultiPartUploadState& state) { (void)state; - if (uri.is_s3()) { + if (uri.is_file()) { + return Status::Ok(); + } else if (uri.is_s3()) { #ifdef HAVE_S3 S3::MultiPartUploadState s3_state; s3_state.part_number = state.part_number; s3_state.upload_id = *state.upload_id; - s3_state.bucket = *state.bucket; - s3_state.key = *state.s3_key; s3_state.st = state.status; for (auto& part : state.completed_parts) { auto rv = s3_state.completed_parts.try_emplace(part.part_number); rv.first->second.SetETag(part.e_tag->c_str()); rv.first->second.SetPartNumber(part.part_number); } - return s3_.set_multipart_upload_state(uri, s3_state); + return s3_.set_multipart_upload_state(uri.to_string(), s3_state); #else return LOG_STATUS(Status_VFSError("TileDB was built without S3 support")); #endif @@ -1906,7 +1908,8 @@ Status VFS::set_multipart_upload_state( #endif } - return Status::Ok(); + return LOG_STATUS( + Status_VFSError("Unsupported URI schemes: " + uri.to_string())); } Status VFS::flush_multipart_file_buffer(const URI& uri) { diff --git a/tiledb/sm/filesystem/vfs.h b/tiledb/sm/filesystem/vfs.h index 12fb71f5684..b3601d6b8ff 100644 --- a/tiledb/sm/filesystem/vfs.h +++ b/tiledb/sm/filesystem/vfs.h @@ -106,8 +106,6 @@ class VFS { uint64_t part_number; optional upload_id; - optional bucket; - optional s3_key; std::vector completed_parts; Status status; }; diff --git a/tiledb/sm/query/writers/global_order_writer.cc b/tiledb/sm/query/writers/global_order_writer.cc index bcabb6f74a1..69ae9fd8b03 100644 --- a/tiledb/sm/query/writers/global_order_writer.cc +++ b/tiledb/sm/query/writers/global_order_writer.cc @@ -1221,7 +1221,11 @@ GlobalOrderWriter::GlobalWriteState* GlobalOrderWriter::get_global_state() { } std::pair> -GlobalOrderWriter::multipart_upload_state() { +GlobalOrderWriter::multipart_upload_state(bool client) { + if (client) { + return {Status::Ok(), global_write_state_->multipart_upload_state_}; + } + auto meta = global_write_state_->frag_meta_; std::unordered_map result; @@ -1239,7 +1243,7 @@ GlobalOrderWriter::multipart_upload_state() { if (!state.has_value()) { return {Status::Ok(), {}}; } - result[uri->to_string()] = std::move(*state); + result[uri->remove_trailing_slash().last_path_part()] = std::move(*state); if (array_schema_.var_size(name)) { auto&& [status, var_uri] = meta->var_uri(name); @@ -1248,7 +1252,8 @@ GlobalOrderWriter::multipart_upload_state() { auto&& [st, var_state] = storage_manager_->vfs()->multipart_upload_state(*var_uri); RETURN_NOT_OK_TUPLE(st, {}); - result[var_uri->to_string()] = std::move(*var_state); + result[var_uri->remove_trailing_slash().last_path_part()] = + std::move(*var_state); } if (array_schema_.is_nullable(name)) { auto&& [status, validity_uri] = meta->validity_uri(name); @@ -1257,7 +1262,8 @@ GlobalOrderWriter::multipart_upload_state() { auto&& [st, val_state] = storage_manager_->vfs()->multipart_upload_state(*validity_uri); RETURN_NOT_OK_TUPLE(st, {}); - result[validity_uri->to_string()] = std::move(*val_state); + result[validity_uri->remove_trailing_slash().last_path_part()] = + std::move(*val_state); } } @@ -1265,8 +1271,19 @@ GlobalOrderWriter::multipart_upload_state() { } Status GlobalOrderWriter::set_multipart_upload_state( - const URI& uri, const VFS::MultiPartUploadState& state) { - return storage_manager_->vfs()->set_multipart_upload_state(uri, state); + const std::string& uri, + const VFS::MultiPartUploadState& state, + bool client) { + if (client) { + global_write_state_->multipart_upload_state_[uri] = state; + return Status::Ok(); + } + + // uri in this case holds only the buffer name + auto absolute_uri = + global_write_state_->frag_meta_->fragment_uri().join_path(uri); + return storage_manager_->vfs()->set_multipart_upload_state( + absolute_uri, state); } } // namespace sm diff --git a/tiledb/sm/query/writers/global_order_writer.h b/tiledb/sm/query/writers/global_order_writer.h index 152620c2360..aeb8f2eb100 100644 --- a/tiledb/sm/query/writers/global_order_writer.h +++ b/tiledb/sm/query/writers/global_order_writer.h @@ -91,6 +91,11 @@ class GlobalOrderWriter : public WriterBase { /** The last hilbert value written. */ uint64_t last_hilbert_value_; + + /** A mapping of buffer names to multipart upload state used by clients + * to track the write state in remote global order writes */ + std::unordered_map + multipart_upload_state_; }; /* ********************************* */ @@ -146,16 +151,30 @@ class GlobalOrderWriter : public WriterBase { /** * Used in serialization to share the multipart upload state * among cloud executors + * + * @param client true if the code is executed from a client context + * @return A mapping of buffer names to VFS multipart upload states read from + * within this instance's `multipart_upload_state_` if the caller is a client, + * or from within the cloud backend internal mappings if the code is executed + * on the rest server. */ std::pair> - multipart_upload_state(); + multipart_upload_state(bool client); /** * Used in serialization of global order writes to set the multipart upload * state in the internal maps of cloud backends + * + * @param uri complete uri of a buffer file or just the buffer name if client + * is true + * @param state VFS multipart upload state to be set + * @param client true if the code is executed from a client context + * @return Status */ Status set_multipart_upload_state( - const URI& uri, const VFS::MultiPartUploadState& state); + const std::string& uri, + const VFS::MultiPartUploadState& state, + bool client); private: /* ********************************* */ diff --git a/tiledb/sm/serialization/array.cc b/tiledb/sm/serialization/array.cc index a531a840719..f6958841d91 100644 --- a/tiledb/sm/serialization/array.cc +++ b/tiledb/sm/serialization/array.cc @@ -200,6 +200,7 @@ Status array_from_capnp( for (auto array_schema_build : entries) { auto schema{array_schema_from_capnp( array_schema_build.getValue(), array->array_uri())}; + schema.set_array_uri(array->array_uri()); all_schemas[array_schema_build.getKey()] = make_shared(HERE(), schema); } @@ -211,6 +212,7 @@ Status array_from_capnp( auto array_schema_latest_reader = array_reader.getArraySchemaLatest(); auto array_schema_latest{array_schema_from_capnp( array_schema_latest_reader, array->array_uri())}; + array_schema_latest.set_array_uri(array->array_uri()); array->set_array_schema_latest( make_shared(HERE(), array_schema_latest)); } diff --git a/tiledb/sm/serialization/fragment_info.cc b/tiledb/sm/serialization/fragment_info.cc index 4f5984ed40a..41b6df06e00 100644 --- a/tiledb/sm/serialization/fragment_info.cc +++ b/tiledb/sm/serialization/fragment_info.cc @@ -279,6 +279,7 @@ Status fragment_info_from_capnp( fragment_info_reader.getArraySchemaLatest(); auto array_schema_latest{ array_schema_from_capnp(array_schema_latest_reader, array_uri)}; + array_schema_latest.set_array_uri(array_uri); fragment_info->array_schema_latest() = make_shared(HERE(), array_schema_latest); } @@ -292,6 +293,7 @@ Status fragment_info_from_capnp( for (auto array_schema_build : entries) { auto schema{ array_schema_from_capnp(array_schema_build.getValue(), array_uri)}; + schema.set_array_uri(array_uri); fragment_info->array_schemas_all()[array_schema_build.getKey()] = make_shared(HERE(), schema); } diff --git a/tiledb/sm/serialization/fragment_metadata.cc b/tiledb/sm/serialization/fragment_metadata.cc index d27cf1b8748..1ca27b070c0 100644 --- a/tiledb/sm/serialization/fragment_metadata.cc +++ b/tiledb/sm/serialization/fragment_metadata.cc @@ -81,7 +81,12 @@ Status fragment_metadata_from_capnp( } } if (frag_meta_reader.hasFragmentUri()) { - frag_meta->fragment_uri() = URI(frag_meta_reader.getFragmentUri().cStr()); + // Reconstruct the fragment uri out of the received fragment name + auto write_version = array_schema->write_version(); + auto frag_dir_uri = ArrayDirectory::generate_fragment_dir_uri( + write_version, array_schema->array_uri().add_trailing_slash()); + auto fragment_name = std::string(frag_meta_reader.getFragmentUri().cStr()); + frag_meta->fragment_uri() = frag_dir_uri.join_path(fragment_name); } frag_meta->has_timestamps() = frag_meta_reader.getHasTimestamps(); frag_meta->has_delete_meta() = frag_meta_reader.getHasDeleteMeta(); @@ -281,7 +286,8 @@ Status fragment_metadata_to_capnp( } } - frag_meta_builder->setFragmentUri(frag_meta.fragment_uri()); + frag_meta_builder->setFragmentUri( + frag_meta.fragment_uri().remove_trailing_slash().last_path_part()); frag_meta_builder->setHasTimestamps(frag_meta.has_timestamps()); frag_meta_builder->setHasDeleteMeta(frag_meta.has_delete_meta()); frag_meta_builder->setHasConsolidatedFooter( diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ index f36767254a8..2a5a3d16e12 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ @@ -7159,17 +7159,17 @@ const ::capnp::_::RawSchema s_cde352fc27e7ca61 = { 1, 28, i_cde352fc27e7ca61, nullptr, nullptr, { &s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { +static const ::capnp::_::AlignedData<86> b_d492b6734d5e3bf5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 245, 59, 94, 77, 115, 182, 146, 212, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 87, 1, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -7178,49 +7178,35 @@ static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { 97, 114, 116, 85, 112, 108, 111, 97, 100, 83, 116, 97, 116, 101, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 24, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 90, 0, 0, 0, + 97, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 164, 0, 0, 0, 2, 0, 1, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 0, 0, 0, 74, 0, 0, 0, + 105, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 0, 0, 0, 3, 0, 1, 0, - 172, 0, 0, 0, 2, 0, 1, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 0, 0, 0, 58, 0, 0, 0, + 113, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 0, 0, 0, 3, 0, 1, 0, - 176, 0, 0, 0, 2, 0, 1, 0, + 108, 0, 0, 0, 3, 0, 1, 0, + 120, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 168, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 1, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 172, 0, 0, 0, 3, 0, 1, 0, - 184, 0, 0, 0, 2, 0, 1, 0, - 5, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 181, 0, 0, 0, 122, 0, 0, 0, + 117, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 0, 0, 0, 3, 0, 1, 0, - 208, 0, 0, 0, 2, 0, 1, 0, + 116, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, 112, 97, 114, 116, 78, 117, 109, 98, 101, 114, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, @@ -7239,22 +7225,6 @@ static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 98, 117, 99, 107, 101, 116, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 115, 51, 75, 101, 121, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 115, 116, 97, 116, 117, 115, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7282,11 +7252,11 @@ static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { static const ::capnp::_::RawSchema* const d_d492b6734d5e3bf5[] = { &s_bde8ebd7b13d8625, }; -static const uint16_t m_d492b6734d5e3bf5[] = {2, 5, 0, 3, 4, 1}; -static const uint16_t i_d492b6734d5e3bf5[] = {0, 1, 2, 3, 4, 5}; +static const uint16_t m_d492b6734d5e3bf5[] = {3, 0, 2, 1}; +static const uint16_t i_d492b6734d5e3bf5[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_d492b6734d5e3bf5 = { - 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 116, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, - 1, 6, i_d492b6734d5e3bf5, nullptr, nullptr, { &s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr } + 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 86, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, + 1, 4, i_d492b6734d5e3bf5, nullptr, nullptr, { &s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<49> b_bde8ebd7b13d8625 = { diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h index 2a4f5bb04de..86a7c11d79c 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h @@ -1310,7 +1310,7 @@ struct MultiPartUploadState { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(d492b6734d5e3bf5, 1, 5) + CAPNP_DECLARE_STRUCT_HEADER(d492b6734d5e3bf5, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -11259,12 +11259,6 @@ class MultiPartUploadState::Reader { inline bool hasUploadId() const; inline ::capnp::Text::Reader getUploadId() const; - inline bool hasBucket() const; - inline ::capnp::Text::Reader getBucket() const; - - inline bool hasS3Key() const; - inline ::capnp::Text::Reader getS3Key() const; - inline bool hasStatus() const; inline ::capnp::Text::Reader getStatus() const; @@ -11323,20 +11317,6 @@ class MultiPartUploadState::Builder { inline void adoptUploadId(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownUploadId(); - inline bool hasBucket(); - inline ::capnp::Text::Builder getBucket(); - inline void setBucket(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initBucket(unsigned int size); - inline void adoptBucket(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownBucket(); - - inline bool hasS3Key(); - inline ::capnp::Text::Builder getS3Key(); - inline void setS3Key(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initS3Key(unsigned int size); - inline void adoptS3Key(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownS3Key(); - inline bool hasStatus(); inline ::capnp::Text::Builder getStatus(); inline void setStatus(::capnp::Text::Reader value); @@ -23901,132 +23881,52 @@ MultiPartUploadState::Builder::disownUploadId() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline bool MultiPartUploadState::Reader::hasBucket() const { - return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) - .isNull(); -} -inline bool MultiPartUploadState::Builder::hasBucket() { - return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) - .isNull(); -} -inline ::capnp::Text::Reader MultiPartUploadState::Reader::getBucket() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::getBucket() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); -} -inline void MultiPartUploadState::Builder::setBucket( - ::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), - value); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::initBucket( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), - size); -} -inline void MultiPartUploadState::Builder::adoptBucket( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), - kj::mv(value)); -} -inline ::capnp::Orphan<::capnp::Text> -MultiPartUploadState::Builder::disownBucket() { - return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); -} - -inline bool MultiPartUploadState::Reader::hasS3Key() const { - return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) - .isNull(); -} -inline bool MultiPartUploadState::Builder::hasS3Key() { - return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) - .isNull(); -} -inline ::capnp::Text::Reader MultiPartUploadState::Reader::getS3Key() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::getS3Key() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); -} -inline void MultiPartUploadState::Builder::setS3Key( - ::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), - value); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::initS3Key( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), - size); -} -inline void MultiPartUploadState::Builder::adoptS3Key( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), - kj::mv(value)); -} -inline ::capnp::Orphan<::capnp::Text> -MultiPartUploadState::Builder::disownS3Key() { - return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); -} - inline bool MultiPartUploadState::Reader::hasStatus() const { - return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) .isNull(); } inline bool MultiPartUploadState::Builder::hasStatus() { - return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) .isNull(); } inline ::capnp::Text::Reader MultiPartUploadState::Reader::getStatus() const { return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } inline ::capnp::Text::Builder MultiPartUploadState::Builder::getStatus() { return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } inline void MultiPartUploadState::Builder::setStatus( ::capnp::Text::Reader value) { ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), value); } inline ::capnp::Text::Builder MultiPartUploadState::Builder::initStatus( unsigned int size) { return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), size); } inline void MultiPartUploadState::Builder::adoptStatus( ::capnp::Orphan<::capnp::Text>&& value) { ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); } inline ::capnp::Orphan<::capnp::Text> MultiPartUploadState::Builder::disownStatus() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } inline bool MultiPartUploadState::Reader::hasCompletedParts() const { - return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) .isNull(); } inline bool MultiPartUploadState::Builder::hasCompletedParts() { - return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) .isNull(); } inline ::capnp::List< @@ -24037,7 +23937,7 @@ MultiPartUploadState::Reader::getCompletedParts() const { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>::get(_reader .getPointerField( - ::capnp::bounded<4>() * + ::capnp::bounded<2>() * ::capnp::POINTERS)); } inline ::capnp::List< @@ -24048,7 +23948,7 @@ MultiPartUploadState::Builder::getCompletedParts() { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>::get(_builder .getPointerField( - ::capnp::bounded<4>() * + ::capnp::bounded<2>() * ::capnp::POINTERS)); } inline void MultiPartUploadState::Builder::setCompletedParts( @@ -24058,7 +23958,7 @@ inline void MultiPartUploadState::Builder::setCompletedParts( ::capnp::_::PointerHelpers<::capnp::List< ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>:: - set(_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + set(_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), value); } inline ::capnp::List< @@ -24069,7 +23969,7 @@ MultiPartUploadState::Builder::initCompletedParts(unsigned int size) { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>:: init( - _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), size); } inline void MultiPartUploadState::Builder::adoptCompletedParts( @@ -24080,7 +23980,7 @@ inline void MultiPartUploadState::Builder::adoptCompletedParts( ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>:: adopt( - _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); } inline ::capnp::Orphan<::capnp::List< @@ -24091,7 +23991,7 @@ MultiPartUploadState::Builder::disownCompletedParts() { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>::disown(_builder .getPointerField( - ::capnp::bounded<4>() * + ::capnp::bounded<2>() * ::capnp::POINTERS)); } diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index d775adac88d..60b08b4039e 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -82,8 +82,6 @@ namespace serialization { #ifdef TILEDB_SERIALIZATION -enum class SerializationContext { CLIENT, SERVER, BACKUP }; - shared_ptr dummy_logger = make_shared(HERE(), ""); Status stats_to_capnp(Stats& stats, capnp::Stats::Builder* stats_builder) { @@ -1150,7 +1148,8 @@ Status delete_to_capnp( Status writer_to_capnp( const Query& query, WriterBase& writer, - capnp::Writer::Builder* writer_builder) { + capnp::Writer::Builder* writer_builder, + bool client_side) { writer_builder->setCheckCoordDups(writer.get_check_coord_dups()); writer_builder->setCheckCoordOOB(writer.get_check_coord_oob()); writer_builder->setDedupCoords(writer.get_dedup_coords()); @@ -1190,7 +1189,7 @@ Status writer_to_capnp( if (globalstate) { auto global_state_builder = writer_builder->initGlobalWriteStateV1(); RETURN_NOT_OK(global_write_state_to_capnp( - query, globalwriter, &global_state_builder)); + query, globalwriter, &global_state_builder, client_side)); } } @@ -1200,7 +1199,8 @@ Status writer_to_capnp( Status writer_from_capnp( const Query& query, const capnp::Writer::Reader& writer_reader, - WriterBase* writer) { + WriterBase* writer, + const SerializationContext context) { writer->set_check_coord_dups(writer_reader.getCheckCoordDups()); writer->set_check_coord_oob(writer_reader.getCheckCoordOOB()); writer->set_dedup_coords(writer_reader.getDedupCoords()); @@ -1225,7 +1225,7 @@ Status writer_from_capnp( RETURN_NOT_OK(global_writer->init_global_write_state()); } RETURN_NOT_OK(global_write_state_from_capnp( - query, writer_reader.getGlobalWriteStateV1(), global_writer)); + query, writer_reader.getGlobalWriteStateV1(), global_writer, context)); } return Status::Ok(); @@ -1346,7 +1346,7 @@ Status query_to_capnp( query_builder->setVarOffsetsAddExtraElement( writer->offsets_extra_element()); query_builder->setVarOffsetsBitsize(writer->offsets_bitsize()); - RETURN_NOT_OK(writer_to_capnp(query, *writer, &builder)); + RETURN_NOT_OK(writer_to_capnp(query, *writer, &builder, client_side)); } else { auto builder = query_builder->initDelete(); auto delete_strategy = dynamic_cast(query.strategy(true)); @@ -1370,7 +1370,9 @@ Status query_to_capnp( auto builder = query_builder->initWrittenFragmentInfo(written_fragment_info.size()); for (uint64_t i = 0; i < written_fragment_info.size(); ++i) { - builder[i].setUri(written_fragment_info[i].uri_); + builder[i].setUri(written_fragment_info[i] + .uri_.remove_trailing_slash() + .last_path_part()); auto range_builder = builder[i].initTimestampRange(2); range_builder.set(0, written_fragment_info[i].timestamp_range_.first); range_builder.set(1, written_fragment_info[i].timestamp_range_.second); @@ -1933,7 +1935,7 @@ Status query_from_capnp( writer->set_offsets_bitsize(query_reader.getVarOffsetsBitsize())); } - RETURN_NOT_OK(writer_from_capnp(*query, writer_reader, writer)); + RETURN_NOT_OK(writer_from_capnp(*query, writer_reader, writer, context)); // For sparse writes we want to explicitly set subarray to nullptr. const bool sparse_write = @@ -1982,8 +1984,13 @@ Status query_from_capnp( auto reader_list = query_reader.getWrittenFragmentInfo(); auto& written_fi = query->get_written_fragment_info(); for (auto fi : reader_list) { + auto write_version = query->array_schema().write_version(); + auto frag_dir_uri = ArrayDirectory::generate_fragment_dir_uri( + write_version, + query->array_schema().array_uri().add_trailing_slash()); + auto fragment_name = std::string(fi.getUri().cStr()); written_fi.emplace_back( - URI(fi.getUri().cStr()), + frag_dir_uri.join_path(fragment_name), std::make_pair(fi.getTimestampRange()[0], fi.getTimestampRange()[1])); } } @@ -2398,7 +2405,8 @@ Status query_est_result_size_deserialize( Status global_write_state_to_capnp( const Query& query, GlobalOrderWriter& globalwriter, - capnp::GlobalWriteState::Builder* state_builder) { + capnp::GlobalWriteState::Builder* state_builder, + bool client_side) { auto& write_state = *globalwriter.get_global_state(); auto& cells_written = write_state.cells_written_; @@ -2453,7 +2461,8 @@ Status global_write_state_to_capnp( state_builder->setLastHilbertValue(write_state.last_hilbert_value_); // Serialize the multipart upload state - auto&& [st, multipart_states] = globalwriter.multipart_upload_state(); + auto&& [st, multipart_states] = + globalwriter.multipart_upload_state(client_side); RETURN_NOT_OK(st); if (!multipart_states.empty()) { @@ -2470,12 +2479,6 @@ Status global_write_state_to_capnp( if (state.upload_id.has_value()) { multipart_entry_builder.setUploadId(*state.upload_id); } - if (state.bucket.has_value()) { - multipart_entry_builder.setBucket(*state.bucket); - } - if (state.s3_key.has_value()) { - multipart_entry_builder.setS3Key(*state.s3_key); - } if (!state.status.ok()) { multipart_entry_builder.setStatus(state.status.message()); } @@ -2501,7 +2504,8 @@ Status global_write_state_to_capnp( Status global_write_state_from_capnp( const Query& query, const capnp::GlobalWriteState::Reader& state_reader, - GlobalOrderWriter* globalwriter) { + GlobalOrderWriter* globalwriter, + const SerializationContext context) { auto write_state = globalwriter->get_global_state(); if (state_reader.hasCellsWritten()) { @@ -2558,16 +2562,10 @@ Status global_write_state_from_capnp( for (auto entry : multipart_reader.getEntries()) { VFS::MultiPartUploadState deserialized_state; auto state = entry.getValue(); - auto uri = URI(entry.getKey().cStr()); + std::string buffer_uri(entry.getKey().cStr()); if (state.hasUploadId()) { deserialized_state.upload_id = state.getUploadId(); } - if (state.hasBucket()) { - deserialized_state.bucket = state.getBucket(); - } - if (state.hasS3Key()) { - deserialized_state.s3_key = state.getS3Key(); - } if (state.hasStatus()) { deserialized_state.status = Status(std::string(state.getStatus().cStr()), ""); @@ -2584,8 +2582,10 @@ Status global_write_state_from_capnp( } } - RETURN_NOT_OK( - globalwriter->set_multipart_upload_state(uri, deserialized_state)); + RETURN_NOT_OK(globalwriter->set_multipart_upload_state( + buffer_uri, + deserialized_state, + context == SerializationContext::CLIENT)); } } } diff --git a/tiledb/sm/serialization/query.h b/tiledb/sm/serialization/query.h index df76ae8eccc..68b36e56351 100644 --- a/tiledb/sm/serialization/query.h +++ b/tiledb/sm/serialization/query.h @@ -195,15 +195,19 @@ Status query_est_result_size_deserialize( #ifdef TILEDB_SERIALIZATION +enum class SerializationContext { CLIENT, SERVER, BACKUP }; + Status global_write_state_to_capnp( const Query& query, GlobalOrderWriter& globalwriter, - capnp::GlobalWriteState::Builder* state_builder); + capnp::GlobalWriteState::Builder* state_builder, + bool client); Status global_write_state_from_capnp( const Query& query, const capnp::GlobalWriteState::Reader& state_reader, - GlobalOrderWriter* globalwriter); + GlobalOrderWriter* globalwriter, + SerializationContext context); Status condition_from_capnp( const capnp::Condition::Reader& condition_reader, diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index a443606eff5..034c5e762e4 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -914,16 +914,10 @@ struct MultiPartUploadState { uploadId@1 :Text; # S3 specific ID identifying a multipart upload process for a file - bucket@2 :Text; - # The S3 bucket name - - s3Key@3 :Text; - # S3 specific multipart upload key - - status@4 :Text; + status@2 :Text; # Status field used to signal an error in a multipart upload process - completedParts@5 :List(CompletedPart); + completedParts@3 :List(CompletedPart); # A list of parts that are already uploaded } struct CompletedPart { diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ index f36767254a8..2a5a3d16e12 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ @@ -7159,17 +7159,17 @@ const ::capnp::_::RawSchema s_cde352fc27e7ca61 = { 1, 28, i_cde352fc27e7ca61, nullptr, nullptr, { &s_cde352fc27e7ca61, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { +static const ::capnp::_::AlignedData<86> b_d492b6734d5e3bf5 = { { 0, 0, 0, 0, 5, 0, 6, 0, 245, 59, 94, 77, 115, 182, 146, 212, 18, 0, 0, 0, 1, 0, 1, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 5, 0, 7, 0, 0, 0, 0, 0, + 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 58, 1, 0, 0, 37, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 33, 0, 0, 0, 87, 1, 0, 0, + 33, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, @@ -7178,49 +7178,35 @@ static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { 97, 114, 116, 85, 112, 108, 111, 97, 100, 83, 116, 97, 116, 101, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 24, 0, 0, 0, 3, 0, 4, 0, + 16, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 153, 0, 0, 0, 90, 0, 0, 0, + 97, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 152, 0, 0, 0, 3, 0, 1, 0, - 164, 0, 0, 0, 2, 0, 1, 0, + 96, 0, 0, 0, 3, 0, 1, 0, + 108, 0, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 161, 0, 0, 0, 74, 0, 0, 0, + 105, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 160, 0, 0, 0, 3, 0, 1, 0, - 172, 0, 0, 0, 2, 0, 1, 0, + 104, 0, 0, 0, 3, 0, 1, 0, + 116, 0, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 169, 0, 0, 0, 58, 0, 0, 0, + 113, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 164, 0, 0, 0, 3, 0, 1, 0, - 176, 0, 0, 0, 2, 0, 1, 0, + 108, 0, 0, 0, 3, 0, 1, 0, + 120, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 173, 0, 0, 0, 50, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 168, 0, 0, 0, 3, 0, 1, 0, - 180, 0, 0, 0, 2, 0, 1, 0, - 4, 0, 0, 0, 3, 0, 0, 0, - 0, 0, 1, 0, 4, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 177, 0, 0, 0, 58, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 172, 0, 0, 0, 3, 0, 1, 0, - 184, 0, 0, 0, 2, 0, 1, 0, - 5, 0, 0, 0, 4, 0, 0, 0, - 0, 0, 1, 0, 5, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 181, 0, 0, 0, 122, 0, 0, 0, + 117, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 180, 0, 0, 0, 3, 0, 1, 0, - 208, 0, 0, 0, 2, 0, 1, 0, + 116, 0, 0, 0, 3, 0, 1, 0, + 144, 0, 0, 0, 2, 0, 1, 0, 112, 97, 114, 116, 78, 117, 109, 98, 101, 114, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, @@ -7239,22 +7225,6 @@ static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 98, 117, 99, 107, 101, 116, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 115, 51, 75, 101, 121, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 115, 116, 97, 116, 117, 115, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -7282,11 +7252,11 @@ static const ::capnp::_::AlignedData<116> b_d492b6734d5e3bf5 = { static const ::capnp::_::RawSchema* const d_d492b6734d5e3bf5[] = { &s_bde8ebd7b13d8625, }; -static const uint16_t m_d492b6734d5e3bf5[] = {2, 5, 0, 3, 4, 1}; -static const uint16_t i_d492b6734d5e3bf5[] = {0, 1, 2, 3, 4, 5}; +static const uint16_t m_d492b6734d5e3bf5[] = {3, 0, 2, 1}; +static const uint16_t i_d492b6734d5e3bf5[] = {0, 1, 2, 3}; const ::capnp::_::RawSchema s_d492b6734d5e3bf5 = { - 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 116, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, - 1, 6, i_d492b6734d5e3bf5, nullptr, nullptr, { &s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr } + 0xd492b6734d5e3bf5, b_d492b6734d5e3bf5.words, 86, d_d492b6734d5e3bf5, m_d492b6734d5e3bf5, + 1, 4, i_d492b6734d5e3bf5, nullptr, nullptr, { &s_d492b6734d5e3bf5, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<49> b_bde8ebd7b13d8625 = { diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h index 2a4f5bb04de..86a7c11d79c 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h @@ -1310,7 +1310,7 @@ struct MultiPartUploadState { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(d492b6734d5e3bf5, 1, 5) + CAPNP_DECLARE_STRUCT_HEADER(d492b6734d5e3bf5, 1, 3) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -11259,12 +11259,6 @@ class MultiPartUploadState::Reader { inline bool hasUploadId() const; inline ::capnp::Text::Reader getUploadId() const; - inline bool hasBucket() const; - inline ::capnp::Text::Reader getBucket() const; - - inline bool hasS3Key() const; - inline ::capnp::Text::Reader getS3Key() const; - inline bool hasStatus() const; inline ::capnp::Text::Reader getStatus() const; @@ -11323,20 +11317,6 @@ class MultiPartUploadState::Builder { inline void adoptUploadId(::capnp::Orphan<::capnp::Text>&& value); inline ::capnp::Orphan<::capnp::Text> disownUploadId(); - inline bool hasBucket(); - inline ::capnp::Text::Builder getBucket(); - inline void setBucket(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initBucket(unsigned int size); - inline void adoptBucket(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownBucket(); - - inline bool hasS3Key(); - inline ::capnp::Text::Builder getS3Key(); - inline void setS3Key(::capnp::Text::Reader value); - inline ::capnp::Text::Builder initS3Key(unsigned int size); - inline void adoptS3Key(::capnp::Orphan<::capnp::Text>&& value); - inline ::capnp::Orphan<::capnp::Text> disownS3Key(); - inline bool hasStatus(); inline ::capnp::Text::Builder getStatus(); inline void setStatus(::capnp::Text::Reader value); @@ -23901,132 +23881,52 @@ MultiPartUploadState::Builder::disownUploadId() { _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); } -inline bool MultiPartUploadState::Reader::hasBucket() const { - return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) - .isNull(); -} -inline bool MultiPartUploadState::Builder::hasBucket() { - return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) - .isNull(); -} -inline ::capnp::Text::Reader MultiPartUploadState::Reader::getBucket() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::getBucket() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); -} -inline void MultiPartUploadState::Builder::setBucket( - ::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), - value); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::initBucket( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), - size); -} -inline void MultiPartUploadState::Builder::adoptBucket( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), - kj::mv(value)); -} -inline ::capnp::Orphan<::capnp::Text> -MultiPartUploadState::Builder::disownBucket() { - return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); -} - -inline bool MultiPartUploadState::Reader::hasS3Key() const { - return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) - .isNull(); -} -inline bool MultiPartUploadState::Builder::hasS3Key() { - return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) - .isNull(); -} -inline ::capnp::Text::Reader MultiPartUploadState::Reader::getS3Key() const { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::getS3Key() { - return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); -} -inline void MultiPartUploadState::Builder::setS3Key( - ::capnp::Text::Reader value) { - ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), - value); -} -inline ::capnp::Text::Builder MultiPartUploadState::Builder::initS3Key( - unsigned int size) { - return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), - size); -} -inline void MultiPartUploadState::Builder::adoptS3Key( - ::capnp::Orphan<::capnp::Text>&& value) { - ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), - kj::mv(value)); -} -inline ::capnp::Orphan<::capnp::Text> -MultiPartUploadState::Builder::disownS3Key() { - return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS)); -} - inline bool MultiPartUploadState::Reader::hasStatus() const { - return !_reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) .isNull(); } inline bool MultiPartUploadState::Builder::hasStatus() { - return !_builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS) + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) .isNull(); } inline ::capnp::Text::Reader MultiPartUploadState::Reader::getStatus() const { return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _reader.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); + _reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } inline ::capnp::Text::Builder MultiPartUploadState::Builder::getStatus() { return ::capnp::_::PointerHelpers<::capnp::Text>::get( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } inline void MultiPartUploadState::Builder::setStatus( ::capnp::Text::Reader value) { ::capnp::_::PointerHelpers<::capnp::Text>::set( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), value); } inline ::capnp::Text::Builder MultiPartUploadState::Builder::initStatus( unsigned int size) { return ::capnp::_::PointerHelpers<::capnp::Text>::init( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), size); } inline void MultiPartUploadState::Builder::adoptStatus( ::capnp::Orphan<::capnp::Text>&& value) { ::capnp::_::PointerHelpers<::capnp::Text>::adopt( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), kj::mv(value)); } inline ::capnp::Orphan<::capnp::Text> MultiPartUploadState::Builder::disownStatus() { return ::capnp::_::PointerHelpers<::capnp::Text>::disown( - _builder.getPointerField(::capnp::bounded<3>() * ::capnp::POINTERS)); + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); } inline bool MultiPartUploadState::Reader::hasCompletedParts() const { - return !_reader.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + return !_reader.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) .isNull(); } inline bool MultiPartUploadState::Builder::hasCompletedParts() { - return !_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS) + return !_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS) .isNull(); } inline ::capnp::List< @@ -24037,7 +23937,7 @@ MultiPartUploadState::Reader::getCompletedParts() const { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>::get(_reader .getPointerField( - ::capnp::bounded<4>() * + ::capnp::bounded<2>() * ::capnp::POINTERS)); } inline ::capnp::List< @@ -24048,7 +23948,7 @@ MultiPartUploadState::Builder::getCompletedParts() { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>::get(_builder .getPointerField( - ::capnp::bounded<4>() * + ::capnp::bounded<2>() * ::capnp::POINTERS)); } inline void MultiPartUploadState::Builder::setCompletedParts( @@ -24058,7 +23958,7 @@ inline void MultiPartUploadState::Builder::setCompletedParts( ::capnp::_::PointerHelpers<::capnp::List< ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>:: - set(_builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + set(_builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), value); } inline ::capnp::List< @@ -24069,7 +23969,7 @@ MultiPartUploadState::Builder::initCompletedParts(unsigned int size) { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>:: init( - _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), size); } inline void MultiPartUploadState::Builder::adoptCompletedParts( @@ -24080,7 +23980,7 @@ inline void MultiPartUploadState::Builder::adoptCompletedParts( ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>:: adopt( - _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS), + _builder.getPointerField(::capnp::bounded<2>() * ::capnp::POINTERS), kj::mv(value)); } inline ::capnp::Orphan<::capnp::List< @@ -24091,7 +23991,7 @@ MultiPartUploadState::Builder::disownCompletedParts() { ::tiledb::sm::serialization::capnp::CompletedPart, ::capnp::Kind::STRUCT>>::disown(_builder .getPointerField( - ::capnp::bounded<4>() * + ::capnp::bounded<2>() * ::capnp::POINTERS)); } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 9826e29fcda..a180ddd6725 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1629,6 +1629,7 @@ StorageManager::load_all_array_schemas( auto&& [st, array_schema] = load_array_schema_from_uri(schema_uri, encryption_key); RETURN_NOT_OK(st); + array_schema.value()->set_array_uri(array_uri); schema_vector[schema_ith] = array_schema.value(); } catch (std::exception& e) { return Status_StorageManagerError(e.what());