Skip to content

Commit

Permalink
CancelAll to also cancel directory upload
Browse files Browse the repository at this point in the history
  • Loading branch information
SergeyRyabinin committed Dec 19, 2023
1 parent 85cf970 commit 31b80c4
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 17 deletions.
4 changes: 2 additions & 2 deletions src/aws-cpp-sdk-core/include/aws/core/platform/FileSystem.h
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ namespace FileSystem

Aws::String path;
Aws::String relativePath;
FileType fileType;
int64_t fileSize;
FileType fileType = FileType::None;
int64_t fileSize = 0;
};

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ static const char* FILE_SYSTEM_UTILS_LOG_TAG = "FileSystemUtils";

AWS_LOGSTREAM_TRACE(FILE_SYSTEM_UTILS_LOG_TAG, "Calling stat on path " << entry.path);

struct stat dirInfo;
struct stat dirInfo = {};
if(!lstat(entry.path.c_str(), &dirInfo))
{
if(S_ISDIR(dirInfo.st_mode))
Expand All @@ -139,7 +139,7 @@ static const char* FILE_SYSTEM_UTILS_LOG_TAG = "FileSystemUtils";
entry.fileType = FileType::File;
}

entry.fileSize = static_cast<int64_t>(dirInfo.st_size);
entry.fileSize = static_cast<unsigned long long>(dirInfo.st_size);
AWS_LOGSTREAM_DEBUG(FILE_SYSTEM_UTILS_LOG_TAG, "file size detected as " << entry.fileSize);
}
else
Expand All @@ -150,7 +150,7 @@ static const char* FILE_SYSTEM_UTILS_LOG_TAG = "FileSystemUtils";
return entry;
}

DIR* m_dir;
DIR* m_dir = nullptr;
};

Aws::String GetHomeDirectory()
Expand Down
12 changes: 6 additions & 6 deletions src/aws-cpp-sdk-transfer/include/aws/transfer/TransferHandle.h
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,17 @@ namespace Aws
void SetChecksum(const Aws::String& checksum) { m_checksum = checksum; }
private:

int m_partId;
int m_partId = 0;

Aws::String m_eTag;
uint64_t m_currentProgressInBytes;
uint64_t m_bestProgressInBytes;
uint64_t m_sizeInBytes;
uint64_t m_rangeBegin;
uint64_t m_currentProgressInBytes = 0;
uint64_t m_bestProgressInBytes = 0;
uint64_t m_sizeInBytes = 0;
uint64_t m_rangeBegin = 0;

std::atomic<Aws::IOStream *> m_downloadPartStream;
std::atomic<unsigned char*> m_downloadBuffer;
bool m_lastPart;
bool m_lastPart = false;
Aws::String m_checksum;
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,13 @@ namespace Aws
* you are using for your client configuration. This executor will be used in a different context than the s3 client is used.
* It is not a bug to use the same executor, but at least be aware that this is how the manager will be used.
*/
Aws::Utils::Threading::Executor* transferExecutor;
Aws::Utils::Threading::Executor* transferExecutor = nullptr;
/**
* When true, TransferManager will calculate the MD5 digest of the content being uploaded.
* The digest is sent to S3 via an HTTP header enabling the service to perform integrity checks.
* This option is disabled by default. Defer to checksumAlgorithm to use other checksum algorithms.
*/
bool computeContentMD5;
bool computeContentMD5 = false;
/**
* If you have special arguments you want passed to our put object calls, put them here. We will copy the template for each put object call
* overriding the body stream, bucket, and key. If object metadata is passed through, we will override that as well.
Expand Down Expand Up @@ -89,12 +89,12 @@ namespace Aws
* allocate for all transfer buffers. default is 50MB.
* If you are using Aws::Utils::Threading::PooledThreadExecutor for transferExecutor, this size should be greater than bufferSize * poolSize.
*/
uint64_t transferBufferMaxHeapSize;
uint64_t transferBufferMaxHeapSize = 10 * MB5;
/**
* Defaults to 5MB. If you are uploading large files, (larger than 50GB, this needs to be specified to be something larger than 5MB. Also keep in mind that you may need
* to increase your max heap size if this is something you plan on increasing.
*/
uint64_t bufferSize;
uint64_t bufferSize = MB5;

/**
* Callback to receive progress updates for uploads.
Expand Down
9 changes: 7 additions & 2 deletions src/aws-cpp-sdk-transfer/source/transfer/TransferManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -276,10 +276,16 @@ namespace Aws
void TransferManager::UploadDirectory(const Aws::String& directory, const Aws::String& bucketName, const Aws::String& prefix, const Aws::Map<Aws::String, Aws::String>& metadata)
{
assert(m_transferConfig.transferInitiatedCallback);
auto handle = Aws::MakeShared<TransferHandle>(CLASS_TAG, bucketName, prefix); // fake handle

auto self = shared_from_this();
auto visitor = [self, bucketName, prefix, metadata](const Aws::FileSystem::DirectoryTree*, const Aws::FileSystem::DirectoryEntry& entry)
auto visitor = [self, bucketName, prefix, metadata, handle](const Aws::FileSystem::DirectoryTree*, const Aws::FileSystem::DirectoryEntry& entry)
{
if (!handle || !handle->ShouldContinue())
{
return false; // Allow to cancel directory upload
}

if (entry && entry.fileType == Aws::FileSystem::FileType::File)
{
Aws::StringStream ssKey;
Expand All @@ -298,7 +304,6 @@ namespace Aws
return true;
};

auto handle = Aws::MakeShared<TransferHandle>(CLASS_TAG, bucketName, prefix); // fake handle
AddTask(handle);
m_transferConfig.transferExecutor->Submit(
[directory, visitor, self, handle]()
Expand Down

0 comments on commit 31b80c4

Please sign in to comment.