Blazor Webassemby Rsa Decryption using Web Crypto API

I've been trying to run this system where I can encrypt in blazor webassembly (server side) and decrypt it in client side using JSInterop, but I can't figure out the error "OperationError" when decrypting the encrypted string. Here's some code.

Generating RSA-OAEP keypair, save the private key in localstorage, and return the public key where blazor server can retrieve.

async function generateKeyPair() {
    if (crypto.subtle) {
        let keyPair = await window.crypto.subtle.generateKey(
            {
                name: "RSA-OAEP",
                modulusLength: 2048,
                publicExponent: new Uint8Array([0x01, 0x00, 0x01]),
                hash: "SHA-256",
            },
            true,
            ["encrypt", "decrypt"],
        );
        const publicKey = await window.crypto.subtle.exportKey("spki", keyPair.publicKey);
        const privateKey = await window.crypto.subtle.exportKey("pkcs8", keyPair.privateKey);

        const stringPublicKey = arrayBufferToBase64(publicKey);
        const stringPrivateKey = arrayBufferToBase64(privateKey);

        localStorage.setItem("browser-privateKey", stringPrivateKey);

        return stringPublicKey;

    } else {
        console.error("Subtle Crypto API is not supported in this browser.");
    }
}

function arrayBufferToBase64(arrayBuffer) {
    let binary = '';
    const bytes = new Uint8Array(arrayBuffer);
    for (let i = 0; i < bytes.byteLength; i++) {
        binary += String.fromCharCode(bytes[i]);
    }
    return btoa(binary);
}

Encrypting plaintext via Blazor Server

        public static string EncryptPayload(string serializedPayload, string publicKey)
        {
            try
            {
                RSAParameters pubkeyRSA = new();

                byte[] keyBytes = Convert.FromBase64String(publicKey);
                using (var rsa = RSA.Create())
                {
                    rsa.ImportSubjectPublicKeyInfo(keyBytes, out _);
                    pubkeyRSA = rsa.ExportParameters(false);
                }

                byte[] payloadBytes = Encoding.UTF8.GetBytes(serializedPayload);
                string encryptedPayload = Convert.ToBase64String(ByteEncrypt(payloadBytes, pubkeyRSA, RsaCryptoHelper.PaddingScheme.RsaOaep)); // using RSA.Create() and RSA.Encrypt(byte[] data, RSAEncryptionPadding.OaepSHA256)

                return encryptedPayload;
            }
            catch (Exception e)
            {
                throw new Exception(e.ToString());
            }
        }

Decrypting encrypted string in Javascript interop

async function decryptData(str) {
    const base64PrivateKey = localStorage.getItem("browser-privateKey");
    const privateKey = await importPrivateKey(base64PrivateKey);
    const encryptedArrayBuffer = new TextEncoder().encode(str);

    return window.crypto.subtle.decrypt(
        {
            name: 'RSA-OAEP'
        },
        privateKey,
        encryptedArrayBuffer
    )
        .then((decryptedArrayBuffer) => {
            const returnString = new TextDecoder().decode(decryptedArrayBuffer);
            return returnString;
        })
        .catch((error) => {
            console.log("Error decrypting data: " + error);
            throw error;
        });
}

async function importPrivateKey(pkcs8Pem) {
    return await window.crypto.subtle.importKey(
        'pkcs8',
        getPkcs8DerDecode(pkcs8Pem),
        {
            name: 'RSA-OAEP',
            hash: 'SHA-256',
        },
        true,
        ['decrypt']
    );
}

function getPkcs8DerDecode(pkcs8Pem) {
    //const pemHeader = '-----BEGIN PRIVATE KEY-----';
    //const pemFooter = '-----END PRIVATE KEY-----';
    //var pemContents = pkcs8Pem.substring(
    //    pemHeader.length,
    //    pkcs8Pem.length - pemFooter.length
    //);
    var binaryDerString = window.atob(pkcs8Pem);
    return str2ab(binaryDerString);
}

I tried to change the keysize and hash value. I also tried changing the padding scheme in c# side. still no luck.

I believe Topaco was on to something in his comment. You are giving your TextEncoder encode method a base64 encoded string, but it does not decode base64. You need to use a base64 to ArrayBuffer decoding method. I give an example I have tested below.

async function decryptData(str) {
    // str here is base64 encoded
    // (encryptedPayload returned from EncryptPayload method on server)
    const base64PrivateKey = localStorage.getItem("browser-privateKey");
    const privateKey = await importPrivateKey(base64PrivateKey);
    // const encryptedArrayBuffer = new TextEncoder().encode(str);
    const encryptedArrayBuffer = base64ToArrayBuffer(str);

    return window.crypto.subtle.decrypt(
        {
            name: 'RSA-OAEP'
        },
        privateKey,
        encryptedArrayBuffer
    )
        .then((decryptedArrayBuffer) => {
            const returnString = new TextDecoder().decode(decryptedArrayBuffer);
            return returnString;
        })
        .catch((error) => {
            console.log("Error decrypting data: " + error);
            throw error;
        });
}

// use method of your choice to convert base64 string to ArrayBuffer 

// Below works but is not as compatible with CSP so added additional method
async function base64ToArrayBufferFetch(base64)
{
    var resp = await fetch(`data:application/octet-binary;base64,${base64}`);
    return await resp.arrayBuffer();
}

// CSP compatible base64 decoder
function base64ToArrayBuffer(base64) { 
    const binaryString = atob(base64); 
    const length = binaryString.length; 
    const arrayBuffer = new ArrayBuffer(length); 
    const uint8Array = new Uint8Array(arrayBuffer); 
    for (let i = 0; i < length; i++) { 
        uint8Array[i] = binaryString.charCodeAt(i); 
    } 
    return arrayBuffer; 
}

I came to this conclusion after writing up some test code as similar to yours as possible and that is the only issue I found.

Update:
I added a method supplied by Jake Rebullo that is compatible with stricter ContentSecurityPolicy setups