Skip to content

Add gzip compression to http payloads above a certain size #24

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

Add gzip compression to http payloads above a certain size #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 4 additions & 38 deletions Drift/Private/DriftEventManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ DECLARE_CYCLE_STAT(TEXT("UploadDriftEvents"), STAT_UploadDriftEvents, STATGROUP_

static constexpr float FLUSH_EVENTS_INTERVAL = 10.0f;
static constexpr int32 MAX_PENDING_EVENTS = 20;
static constexpr int32 MIN_SIZE_PAYLOAD_TO_COMPRESS = 200;

FDriftEventManager::FDriftEventManager()
{
Expand Down Expand Up @@ -98,32 +97,11 @@ void FDriftEventManager::FlushEvents()
FString Payload;
JsonArchive::SaveObject(Events, Payload);

TArray<uint8> Compressed;
bool bUseCompressed = false;
const FTCHARToUTF8 Converter(*Payload);
const auto UncompressedSize{ Converter.Length() };
if (UncompressedSize >= MIN_SIZE_PAYLOAD_TO_COMPRESS)
{
UE_LOG(LogDriftEvent, Verbose, TEXT("Attempting to compress payload"));

Compressed.SetNumUninitialized(UncompressedSize);
const auto Uncompressed = reinterpret_cast<const uint8*>(Converter.Get());
int32 CompressedSize;
const auto CompressionResult = FCompression::CompressMemory(NAME_Gzip, Compressed.GetData(), CompressedSize, Uncompressed, UncompressedSize);
if (CompressionResult && CompressedSize < UncompressedSize)
{
UE_LOG(LogDriftEvent, Verbose, TEXT("Payload compression size is smaller than uncompressed size. Using compressed payload."));

Compressed.SetNum(CompressedSize);
bUseCompressed = true;
}
}

const auto EndTime = FPlatformTime::Seconds();

UE_LOG(LogDriftEvent, Verbose, TEXT("Processed '%d' events in '%.3f' seconds"), Events.Num(), EndTime - StartTime);

AsyncTask(ENamedThreads::GameThread, [WeakSelf, Payload, Compressed, bUseCompressed]()
AsyncTask(ENamedThreads::GameThread, [WeakSelf, Payload]()
{
SCOPE_CYCLE_COUNTER(STAT_UploadDriftEvents);

Expand All @@ -133,30 +111,18 @@ void FDriftEventManager::FlushEvents()

if (!PinnedSelf.IsValid())
{
UE_LOG(LogDriftEvent, Error, TEXT("Failed to flush events. The event manager became invalid during processing."));
UE_LOG(LogDriftEvent, Warning, TEXT("Failed to flush events. The event manager became invalid during processing."));
return;
}

const auto RequestManager = PinnedSelf->requestManager.Pin();
if (!RequestManager.IsValid())
{
UE_LOG(LogDriftEvent, Error, TEXT("Failed to flush events. Request manager became invalid during processing."));
UE_LOG(LogDriftEvent, Warning, TEXT("Failed to flush events. Request manager became invalid during processing."));
return;
}

const auto Request = RequestManager->CreateRequest(HttpMethods::XPOST, PinnedSelf->eventsUrl, HttpStatusCodes::Created);

if (bUseCompressed)
{
Request->SetContent(Compressed);
Request->SetHeader(TEXT("Content-Encoding"), TEXT("gzip"));
}
else
{
Request->SetPayload(Payload);
}

Request->SetHeader(TEXT("Content-Type"), TEXT("application/json"));
const auto Request = RequestManager->Post(PinnedSelf->eventsUrl, Payload);
Request->Dispatch();
});
});
Expand Down
48 changes: 44 additions & 4 deletions DriftHttp/Private/HttpRequest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

#define LOCTEXT_NAMESPACE "Drift"

static constexpr int32 MIN_SIZE_PAYLOAD_TO_COMPRESS = 200;

HttpRequest::HttpRequest()
: MaxRetries_{ 0 }
Expand Down Expand Up @@ -518,15 +519,54 @@ FString HttpRequest::GetContentAsString() const

void HttpRequest::SetPayload(const FString& content)
{
if (!content.IsEmpty())
if (content.IsEmpty())
{
wrappedRequest_->SetContentAsString(content);
return;
}

// Set Content-Type
SetHeader(TEXT("Content-Type"), contentType_);

// Attempt to compress the payload if above the threshold
const FTCHARToUTF8 Converter(*content);
const auto UncompressedSize{ Converter.Length() };
if (UncompressedSize >= MIN_SIZE_PAYLOAD_TO_COMPRESS)
{
UE_LOG(LogHttpClient, Verbose, TEXT("Attempting to compress payload of size '%d' bytes since it is above the compression threshold size of '%d' bytes"), UncompressedSize, MIN_SIZE_PAYLOAD_TO_COMPRESS);

if (content.Len())
TArray<uint8> Compressed;
auto CompressedSize = FCompression::CompressMemoryBound(NAME_Gzip, UncompressedSize);
Compressed.SetNumUninitialized(CompressedSize);

const auto Uncompressed = reinterpret_cast<const uint8*>(Converter.Get());

const auto CompressionResult = FCompression::CompressMemory(NAME_Gzip, Compressed.GetData(), CompressedSize, Uncompressed, UncompressedSize);

if (CompressionResult)
{
SetHeader(TEXT("Content-Type"), contentType_);
if (CompressedSize < UncompressedSize)
{
UE_LOG(LogHttpClient, Verbose, TEXT("Compression size is smaller than uncompressed size. Using compressed payload. Compressed size: '%d', Uncompressed size: '%d', Ratio: '%.2f'"),
CompressedSize, UncompressedSize, static_cast<float>(CompressedSize) / static_cast<float>(UncompressedSize));

UE_LOG(LogHttpClient, VeryVerbose, TEXT("Compressed payload:\n'%s'"), *BytesToString(Compressed.GetData(), CompressedSize));

Compressed.SetNum(CompressedSize);
wrappedRequest_->SetContent(Compressed);
wrappedRequest_->SetHeader(TEXT("Content-Encoding"), TEXT("gzip"));

return;
}

UE_LOG(LogHttpClient, Verbose, TEXT("Compression didn't change the size of the payload. Using uncompressed payload."));
}
else
{
UE_LOG(LogHttpClient, Verbose, TEXT("Compression failed. Using uncompressed payload."));
}
}

wrappedRequest_->SetContentAsString(content);
}


Expand Down
1 change: 0 additions & 1 deletion DriftHttp/Public/HttpRequest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,6 @@ class DRIFTHTTP_API HttpRequest : public TSharedFromThis<HttpRequest>

void SetExpectJsonResponse(bool expectJsonResponse) { expectJsonResponse_ = expectJsonResponse; }

/** Used by the request manager to set the payload */
void SetPayload(const FString& content);

FString GetAsDebugString(bool detailed = false) const;
Expand Down