How to properly save and restore thread context on 64 bit process (Windows)?

I've got this code below for releasing library from some 64 bit process. It does its job, but the problem is that after restoring saved context, the target process just crashes. Dunno what is the issue here. It should set all registers and flags for what they were before, right?. What am I doin' wrong?

#ifdef _WIN64

const static unsigned char FreeLibrary_InjectionCodeRAW_x64[] =
{
    0x48, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //mov rax, value
    0x48, 0xB9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //mov rcx, value
    0xFF, 0xD0, //call rax (FreeLibrary)
    0x48, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //mov rax, value
    0xC7, 0x00, 0x01, 0x00, 0x00, 0x00, //mov [rax],1
    0x48, 0xB8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //mov rax, value
    0xB9, 0x64, 0x00, 0x00, 0x00, //mov ecx, 0x64
    0xFF, 0xD0, //call Sleep 
    0xEB, 0xED, //jmp
    0x00, 0x00, 0x00, 0x00 //status
};

#pragma pack(push, 1)
struct FreeLibrary_InjectionCode_x64
{
    FreeLibrary_InjectionCode_x64()
    {
        memcpy(this, FreeLibrary_InjectionCodeRAW_x64, sizeof(FreeLibrary_InjectionCodeRAW_x64));
    }

    char code_1[2];
    FARPROC lpFreeLibrary;
    char code_2[2];
    HMODULE hLib;
    char code_3[4];
    LPVOID lpStatusAddress;
    char code_4[8];
    FARPROC lpSleep;
    char code_5[9];
    int status;
};
#pragma pack(pop)

#endif

void FreeLib(const char what[], const char where[])
{
    HANDLE hToken;
    OpenProcessToken(GetCurrentProcess(), TOKEN_ALL_ACCESS, &hToken);
    SetPrivilege(hToken, SE_DEBUG_NAME, TRUE);
    CloseHandle(hToken);
    OpenThreadToken(GetCurrentThread(), TOKEN_ALL_ACCESS, FALSE, &hToken);
    SetPrivilege(hToken, SE_DEBUG_NAME, TRUE);
    CloseHandle(hToken);

    HMODULE hMod;
    DWORD dwProcessId = GetProcessIdByName(where);
    if ((hMod = GetModuleHandleInProcess(what, dwProcessId)) != NULL)
    {
        HANDLE hProcess = OpenProcess(PROCESS_CREATE_THREAD | PROCESS_VM_OPERATION | PROCESS_VM_READ | PROCESS_VM_WRITE | SYNCHRONIZE, FALSE, dwProcessId);
        if (hProcess != NULL)
        {
            HMODULE hKernel = LoadLibrary("kernel32.dll");
            FARPROC FLaddr = GetProcAddress(hKernel, "FreeLibrary");
            FARPROC Saddr = GetProcAddress(hKernel, "Sleep");

            HANDLE hThread = OpenThread(THREAD_GET_CONTEXT | THREAD_SET_CONTEXT | THREAD_QUERY_INFORMATION | THREAD_SET_INFORMATION | THREAD_SUSPEND_RESUME,
                FALSE, GetValidThreadIdInProcess(dwProcessId));

            if (hThread != NULL && FLaddr != NULL && Saddr != NULL)
            {
                LPVOID addr = VirtualAllocEx(hProcess, NULL, 4096, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
                LPVOID lpStatusAddress = (PUCHAR)addr + (sizeof(FreeLibrary_InjectionCode_x64)-sizeof(int));
                FreeLibrary_InjectionCode_x64 code = FreeLibrary_InjectionCode_x64();
                code.hLib = hMod;
                code.lpFreeLibrary = FLaddr;
                code.lpSleep = Saddr;
                code.lpStatusAddress = lpStatusAddress;
                WriteProcessMemory(hProcess, addr, &code, sizeof(FreeLibrary_InjectionCode_x64), NULL);

                CONTEXT ctx, oldCtx;
                ctx.ContextFlags = CONTEXT_ALL;

                SuspendThread(hThread);
                GetThreadContext(hThread, &ctx);

                memcpy(&oldCtx, &ctx, sizeof(CONTEXT));
                ctx.Rip = (DWORD64)addr; 

                SetThreadContext(hThread, &ctx);
                ResumeThread(hThread);

                while (!code.status)
                {
                    Sleep(15);
                    ReadProcessMemory(hProcess, addr, &code, sizeof(FreeLibrary_InjectionCode_x64), NULL);
                }

                SuspendThread(hThread);
                SetThreadContext(hThread, &oldCtx);
                ResumeThread(hThread);

                VirtualFreeEx(hProcess, addr, 4096, MEM_DECOMMIT);

                CloseHandle(hThread);
            }

            CloseHandle(hProcess);
        }
    }
}

Windows 64-bit uses the fastcall calling convention. In this convention the caller of a function is responsible for reserving 4 * 64 bit (32 byte) on the stack for the called function to save registers. This means your calls should look like this:

sub rsp, 32
call rax
add rsp, 32

In your code your calls to FreeLibrary or Sleep overwrite stack that doesn't belong to their stack frame, causing a crash later on.