Decrypt an RSA message from browser with window.crypto.subtle APIs

I'm trying to decode an RSA 2048 bit message encoded with a public key using the corresponding private key. The environment is google chrome and I'm using the window.crypto.subtle APIs.

I generated the key couple and encoded the message using openssl tools:

# generate keys and put the private one in file private_key.pem
openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048

# extract public key in file public_key.pem
openssl rsa -pubout -in private_key.pem -out public_key.pem

# encode message input.txt using the public key
openssl rsautl -encrypt -oaep -inkey public_key.pem -pubin -in input.txt -out msg_rsa.enc

# convert the encoded msg in base 64 format
base64 msg_rsa.enc > msg_rsa_64.enc

This is the javascript code I'm using to decode the message:

function str2ab(str) {
    const buf = new ArrayBuffer(str.length);
    const bufView = new Uint8Array(buf);
    for (let i = 0, strLen = str.length; i < strLen; i++) {
        bufView[i] = str.charCodeAt(i);
    }
    return buf;
}

async function importPrivateKey(pem) {
    pem = pem.replace( /[\r\n]+/gm, "" );
    // fetch the part of the PEM string between header and footer
    const pemHeader = "-----BEGIN PRIVATE KEY-----";
    const pemFooter = "-----END PRIVATE KEY-----";
    const pemContents = pem.substring(pemHeader.length, pem.length - pemFooter.length);
    // base64 decode the string to get the binary data
    const binaryDerString = window.atob(pemContents);
    // convert from a binary string to an ArrayBuffer
    const binaryDer = str2ab(binaryDerString);

    return window.crypto.subtle.importKey(
      "pkcs8",
      binaryDer,
      {
        name: "RSA-OAEP",
        modulusLength: 2048,
        publicExponent: new Uint8Array([1, 0, 1]),
        hash: "SHA-256",
      },
      true,
      ["decrypt"]
    );
}

async function decryptRSA(_key, ciphertext) {
    let decrypted = await window.crypto.subtle.decrypt(
      {
        name: "RSA-OAEP"
      },
      _key,
      ciphertext
    );
    const dec = new TextDecoder();
    return dec.decode(decrypted);
}

const fromBase64 = base64String => Uint8Array.from(atob(base64String), c => c.charCodeAt(0));

window.onload = init;

async function init() {

    const privateKey = '-----BEGIN PRIVATE KEY-----\
MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC3jmTi3O1k2YXs\
AM6nNTTIzDq5YWkxYrYb6cpO9eYuzmphgRnVDR6a1YWRXMoCuCfuNXcDGywzudRn\
bBMw0FHKLUqCttVHGpZYu0+0tRR10ubxiz/xnd/aCmRYHcmUNn8Qdh3KU59A9HK5\
HhYFf1vhK8r3fkoO4CjoGo1ROzXyMybUSy+4mSNscUtt5LwrVn48vXvG5i5B4DRT\
nM4cINmutEzA2s5cDt+dzU4Py71fKBRDRIGGn0vdVSoZKbWuhm5WewyRewCk7HFc\
PALCi5/1A7VKDAHUC4FlXmuG2+wzdchEyxMj6oLR7+BkKFQaTmuMM/22cGBjVTVt\
pSr3iDovAgMBAAECggEBAIuTQW+oovNu3IDq1DkdIjgV5AmW4tBkySlMi0OjhBbP\
auEdtDDnOwBtoJU6Q3nx4psmGItKHEBw6+yAp88UeT0NV30x3delhfGO7Trx/s7h\
Qi8lvcfSTqeUA11luSR0lAZGaryw/YX820eccw5XG9yK2ll7tIC/PxvPJOpB5fF2\
XGxGrionTjHDzXJ1OWX0i0aZlNNufInJAHhlt7aT3GiQMKcQs+AUb/+bWxI3Hln8\
KcL13EUlD4pJW8vtTK3gCnQNKKMoPB5Ugqe5BrU8ElkBz+zSKDnVwt5bgjrlucYz\
rKJxWr6/qTRZkzmvkhaJeNzzepfwkFsQ/eHcxYrtuDECgYEA8OXkQ2SqYDZwizCd\
SuVkx2zHm3zXYRSlRtnXYoUdJyTyuZ4k2GvXBrlwRsOJ14emVkHKyR5enmNjwrW5\
dcD2tbBzavcqOYAMiHcKklcS/gWgPx0X5QFHU33vr8u51BQWCz75lxddWNKxVAN/\
cUTugONtS4+EP4dSZhuxHt6RscsCgYEAwxA9QmZrI54hjMkIyqwmviRmIk42S5kk\
OxbhxBbt5qVueLRB092JyGmCR2kYiqdHCYkGFDOE4kni6Bsszq5CSJvhiATFeZeX\
ldFQeZqAiRY2rAd7xD1upMug/cK3ODA8k3k/e72CtyxtBTR01q29SnPx5p/57MrI\
3ogddHlGvK0CgYEA3VqhELwjQh1D9OJK5lM683SlRd7FGdOauyvYmhKu4xU0ZBNI\
0ATnpKoo3R04P+/JjGEQMRXS4794H6ZUMDuLdxAYPiW3ivZ6jbq04BtavEf3I4dc\
OXWfULzbzbFpo9KBHvxS4974S3Hut8AvDqnEbnKML25EmwuBT4oKis8BGVkCgYEA\
nusPDZbFeNou+TUbzYrdcZHUB+TyhTq58s4clxYbMgrbaslozAQ0aavT8Pvle6j2\
zgTth+3FOFr72x+wrJ358I/W+Wrxu7NOU0eZqci/KXCIkDT0l5d5GhewDK3jeYqK\
/5cLqnNmGHfARjpLak9X5V162erBwjIf3nTEkozvnW0CgYB6L1CX3DkTFH3OBcJe\
SvV18RDUfNI8MpUKcpofmwwvgER3GrehSZHRVxVwNbnJOqbh/oiwmmNJieKrFsk9\
EzCRBVWdZByHHYW2js+gCrAp+ghnl1QEAeCU7YTxCJ2fZIAmfB9X4u/7ARtVxnZY\
mOWlm65KUYr5lf2Ws5plL4pCRA==\
-----END PRIVATE KEY-----';

    const ciphertext = 'F6/NwENdUZSl+vrgpWVkyWPQuYaTGDNZPIvj4KmIRHVx4qybxN24LPIgk0Rl84KHcLFadZWCjNpM\
vg3l826OaKZAtwvIp9IxVrMbvtNOymY6A1koKvC9ema92SR4DC9hmTtMxhUB6r3XgACtRLFqMfg+\
zYSHfFqQEGJg3yZ43hfzIq/gCfHPk5sZXASq5WY5b9yd4gRonn5D4OCD6xna/r5ovHfrpO/Fwe8N\
eeY2gqTAdtzvtmOw/HLQhGANejpJYr1IriQbepM7jLjBkJX+uCn38O1MxpQb7s5RXTvGvoEoofWV\
Cq8gNFhgnVFuurdZUiY0bn58UwaVFdwzEfDSUQ==';

    try {
        const key = await importPrivateKey(privateKey);
        const decoded = await decryptRSA(key, fromBase64(ciphertext));
        console.log(decoded);
    } catch(error) {
        console.log(error);
    }
}

Running the code, I got an exception in window.crypto.subtle.decrypt with the rather useless message "DOMException".

What am I doing wrong?

Thanks

There's only one flaw: The posted code currently uses OAEP with SHA256. The ciphertext can be decrypted with the posted key if OAEP with SHA1 is applied as padding.

In addition, the function fromBase64 can be used to Base64 decode the key into a TypedArray, so the function str2ab is actually not needed (but of course this is not an error, just redundant).

const fromBase64 = base64String => Uint8Array.from(atob(base64String), c => c.charCodeAt(0));
const getPkcs8Der = pkcs8Pem => {
    pkcs8Pem = pkcs8Pem.replace( /[\r\n]+/gm, "" );
    const pkcs8PemHeader = "-----BEGIN PRIVATE KEY-----";
    const pkcs8PemFooter = "-----END PRIVATE KEY-----";
    pkcs8Pem = pkcs8Pem.substring(pkcs8PemHeader.length, pkcs8Pem.length - pkcs8PemFooter.length);
    return fromBase64(pkcs8Pem);	
}		
	 
async function importPrivateKey(pkcs8Pem) {		
    return await window.crypto.subtle.importKey(
        "pkcs8",
        getPkcs8Der(pkcs8Pem),
        {
            name: "RSA-OAEP",
            hash: "SHA-1",          // Replace SHA-256 with SHA-1
        },
        true,
        ["decrypt"]
    );
}

async function decryptRSA(key, ciphertext) {
    let decrypted = await window.crypto.subtle.decrypt(
        {
            name: "RSA-OAEP"
        },
        key,
        ciphertext
    );
    const dec = new TextDecoder();
    return dec.decode(decrypted);
}

window.onload = init;

async function init() {

    const privateKey = 
        '-----BEGIN PRIVATE KEY-----\
        MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC3jmTi3O1k2YXs\
        AM6nNTTIzDq5YWkxYrYb6cpO9eYuzmphgRnVDR6a1YWRXMoCuCfuNXcDGywzudRn\
        bBMw0FHKLUqCttVHGpZYu0+0tRR10ubxiz/xnd/aCmRYHcmUNn8Qdh3KU59A9HK5\
        HhYFf1vhK8r3fkoO4CjoGo1ROzXyMybUSy+4mSNscUtt5LwrVn48vXvG5i5B4DRT\
        nM4cINmutEzA2s5cDt+dzU4Py71fKBRDRIGGn0vdVSoZKbWuhm5WewyRewCk7HFc\
        PALCi5/1A7VKDAHUC4FlXmuG2+wzdchEyxMj6oLR7+BkKFQaTmuMM/22cGBjVTVt\
        pSr3iDovAgMBAAECggEBAIuTQW+oovNu3IDq1DkdIjgV5AmW4tBkySlMi0OjhBbP\
        auEdtDDnOwBtoJU6Q3nx4psmGItKHEBw6+yAp88UeT0NV30x3delhfGO7Trx/s7h\
        Qi8lvcfSTqeUA11luSR0lAZGaryw/YX820eccw5XG9yK2ll7tIC/PxvPJOpB5fF2\
        XGxGrionTjHDzXJ1OWX0i0aZlNNufInJAHhlt7aT3GiQMKcQs+AUb/+bWxI3Hln8\
        KcL13EUlD4pJW8vtTK3gCnQNKKMoPB5Ugqe5BrU8ElkBz+zSKDnVwt5bgjrlucYz\
        rKJxWr6/qTRZkzmvkhaJeNzzepfwkFsQ/eHcxYrtuDECgYEA8OXkQ2SqYDZwizCd\
        SuVkx2zHm3zXYRSlRtnXYoUdJyTyuZ4k2GvXBrlwRsOJ14emVkHKyR5enmNjwrW5\
        dcD2tbBzavcqOYAMiHcKklcS/gWgPx0X5QFHU33vr8u51BQWCz75lxddWNKxVAN/\
        cUTugONtS4+EP4dSZhuxHt6RscsCgYEAwxA9QmZrI54hjMkIyqwmviRmIk42S5kk\
        OxbhxBbt5qVueLRB092JyGmCR2kYiqdHCYkGFDOE4kni6Bsszq5CSJvhiATFeZeX\
        ldFQeZqAiRY2rAd7xD1upMug/cK3ODA8k3k/e72CtyxtBTR01q29SnPx5p/57MrI\
        3ogddHlGvK0CgYEA3VqhELwjQh1D9OJK5lM683SlRd7FGdOauyvYmhKu4xU0ZBNI\
        0ATnpKoo3R04P+/JjGEQMRXS4794H6ZUMDuLdxAYPiW3ivZ6jbq04BtavEf3I4dc\
        OXWfULzbzbFpo9KBHvxS4974S3Hut8AvDqnEbnKML25EmwuBT4oKis8BGVkCgYEA\
        nusPDZbFeNou+TUbzYrdcZHUB+TyhTq58s4clxYbMgrbaslozAQ0aavT8Pvle6j2\
        zgTth+3FOFr72x+wrJ358I/W+Wrxu7NOU0eZqci/KXCIkDT0l5d5GhewDK3jeYqK\
        /5cLqnNmGHfARjpLak9X5V162erBwjIf3nTEkozvnW0CgYB6L1CX3DkTFH3OBcJe\
        SvV18RDUfNI8MpUKcpofmwwvgER3GrehSZHRVxVwNbnJOqbh/oiwmmNJieKrFsk9\
        EzCRBVWdZByHHYW2js+gCrAp+ghnl1QEAeCU7YTxCJ2fZIAmfB9X4u/7ARtVxnZY\
        mOWlm65KUYr5lf2Ws5plL4pCRA==\
        -----END PRIVATE KEY-----';

    const ciphertext =
        'F6/NwENdUZSl+vrgpWVkyWPQuYaTGDNZPIvj4KmIRHVx4qybxN24LPIgk0Rl84KHcLFadZWCjNpM\
        vg3l826OaKZAtwvIp9IxVrMbvtNOymY6A1koKvC9ema92SR4DC9hmTtMxhUB6r3XgACtRLFqMfg+\
        zYSHfFqQEGJg3yZ43hfzIq/gCfHPk5sZXASq5WY5b9yd4gRonn5D4OCD6xna/r5ovHfrpO/Fwe8N\
        eeY2gqTAdtzvtmOw/HLQhGANejpJYr1IriQbepM7jLjBkJX+uCn38O1MxpQb7s5RXTvGvoEoofWV\
        Cq8gNFhgnVFuurdZUiY0bn58UwaVFdwzEfDSUQ==';

    try {
        const key = await importPrivateKey(privateKey);
        const decrypted = await decryptRSA(key, fromBase64(ciphertext));
        console.log(decrypted);
    } catch(error) {
        console.log(error);
    }
}