javascriptsslencryptionopensslwebcrypto-api

Web Crypto API decryption failing with RSA-OAEP


I have generated a private key as follows:

openssl genpkey -algorithm RSA -out private_key.pem -pkeyopt rsa_keygen_bits:2048

openssl rsa -pubout -in private_key.pem -out public_key.pem

openssl pkcs8 -topk8 -inform PEM -outform PEM -in private_key.pem -out private_key_pkcs8.pem -nocrypt

The private key is embedded in the code snippet. Then I encrypted a plaintext like the following:

echo -n "abcdef" | openssl rsautl -encrypt -pubin -inkey public_key.pem | openssl base64 > enc_simple.txt

This is also embedded in the code snippet. If I do the following:

base64 -d enc_simple.txt > enc_simple.bin
openssl rsautl -decrypt -inkey private_key_pkcs8.pem -in enc_simple.bin -out decrypted_simple.txt

I get back "abcdef". So with openssl everything is working. Then I want to recreate this entire scenario using Web Crypto API but the decryption is failing. Importing the private key is ok but the rest fails with OperationError with no error message. It is extremely obscure and hard to debug.

I have also run:

od -An -vtu1 enc_simple.bin

to see the Uint8Array values and compare them with what I get from str2ab from the code snippet. They match!

So I am stuck with no error message. I am trying to understand why the decryption is failing.

const encryptedData = `gLiN3BLWTxbwpE5bwcFiB7Y/nK0H4iz9lK0sehMIJtoAPibDPZ8EYp8EnzsGFlKC
+GXse6Ka5YC7UKYn7xUvfBbCDSY7RJ/J9oAzpCEHvCnFOmrsvUtNEhP4w3LTaKL/
qKmjQZaPlZNWCmrzrW3g2A6DUCqLx5EQwCcwME2WjYVhuMZfZOKqtp+uukJMUf3h
cC9J8QkPCk1NZogCad29b7q7JLN1uWEQzOUgB6BqJfAp8kL92dftOy2gpWoGYraI
YPU3Tmp12txMyUY0yKsr4c2dOzxwWUW+ZssVkc6ZGqFVufADziZDp2sfFaz8yBuI
qek7S0KXd6A7qWBYBrzzyw==`; //abcdef
const privKeyPEM=`-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDJiQlvlYbTqi/W
bLa7nzaF4Wz+ypQa01HD+UMHX25Uv3n6P69ucKs/OOdWbcenknAmSBK/VzgIsjlm
vak3y1LoYImMhm8R26Arv8Fv7jwQkbBxVnOn6yfTL2h1SRK+zFzWumfbl2vp7POG
s7CRQaK0HFPKroVlYHlinTlYXSlFqdaknH2ip1J2YPJ3O4w0/ME8gBZ0Mj1XkokB
Qc0Lk76BpvfTuBbzvLYI4bq0/jSoM6ANecqK+T/7/aSFxh2x8acJ/xzJg2H7I+de
pXZ1M9Mz0x94jWRfc9Q9qLgY6AuX7MUyb2YEs+vMxcim60fvktHAaXvn9vM0eauC
B+hpThohAgMBAAECggEAINub0zqAwe/EXuRYopxhqlBHkf77SKhdc2MnX4NanKyf
OYK6mnn6IZOoe/noDFUevc8QZ2vT8e8E1tBjT3px6PscUfH1F+dD5P4djp874cOv
Dbt3ndAELTVUhZLFYKA3HrdDiZTVfk0oozSWvAgEe/MGYkwz6YRiJgbWO6bsvLOa
5Ehhd7Ocxbs4B389ad3bVzRkk02/IRDEeO1z0+22Wv1NIe0Spzl6FVLspL6JdmEu
60rpShfwN7D0G4TQYYp/9Sh+zt0ThIF4kqQoURx5E2uQaDUAT206itMG2mUWYSR1
M37U+46kpWBz0jDaRFmqlySKrmkbx7kWXju8wP1ZAQKBgQDvlq4B3jAPd+Pu/Rxw
GORV5S8PdJrkQnx8BeAviGAVc2w6EMyRWusVzL95mpRS0t0E5xZtB2kLkVU/I08b
j+WqrGFqFp8niXYnJsien8nfLct0AhJVvVaPtt8lHP9SKniBOZc0owzFljnt+4/6
M4ifAiKvJYOqPSdGpJgvJq4TsQKBgQDXVxJCjUNInb4p0WWljxdcMpgSDPn53+BG
M/MrxVbTOvHhKsbNlglYxT/87J+8lFKS5Y9EkpqekCY1aGJ0PYfLBMowvNAwiP2s
89++PTuEO/hK16Xb8znZ7rUGdJdRUitBN8iVkWNLA2pzalcZkkHbIWQbDuJOPbdO
bo1u06U5cQKBgAgeliUQD5bmnD3kLAuMfGiAzNh8PieQLUHSvSc/OupfMALDwPsI
FsF1X+PSHka0SLM61aK6RpASy83I94xakxD2qJJ808X2PZ/UC6Z8ic3bcnKrA04O
jZlvPB643do+ADl45yvsfqlPjwUGqnlzN6UT4HMJFW42hlc5isLGT83xAoGBANT/
UxhxEfRp0wcaECjKeJjBkpmILFp0jynhiM3qzA7zZv0JissfdO9RbBGJHBczvtl+
J0/0kuv0OVbqgTfpBMBTZIsAuAzJ8+F2+AD8IDqT9uxQkcYVt0tRSc2w1VuioxZH
TyhiPoycPFcdADpS6MEPLi11c3NgqEf0IgFVZ0CBAoGBAMybGC+IImSDpZzLaSpz
KAD6Vyto7eK4pENH8bPcM2Hpy7aMGJkr6Q/NT34p/vSfFG2gm8ueILhvqQ8yB6h6
gtfHOEu7CC/8WL4Y/WehrpD+fTTSOd/C+JZJbXjHjJ55NX1JzgeCPWJP2QOyXE85
VLq+2V8bhsQxi4JylnH6suLB
-----END PRIVATE KEY-----`
const
    atob = (window || self)?.atob,
    subtle = (window || self).crypto.subtle,
    trimWs = (s) => s.replace(/\s*/g,""),
    trimHeader = (s) => s.replace(/-----(?:BEGIN|END) PRIVATE KEY-----/gi, "");
async function importPrivateKey (pemkey) {
    const binaryKey = atob(trimWs(trimHeader(pemkey)));
    const keyBuffer = new Uint8Array(binaryKey.length);
    for (let i = 0; i < binaryKey.length; i++) {
        keyBuffer[i] = binaryKey.charCodeAt(i);
    }
    return await subtle.importKey(
        "pkcs8",
        keyBuffer,
        { name: "RSA-OAEP", hash: "SHA-256"},
        false,
        ["decrypt"]
    );
}
async function decryptData (privateKey, encryptedData) {
      const decrypted = await subtle.decrypt(
          { name: "RSA-OAEP", hash: "SHA-256"},
          privateKey,
         str2ab(atob(trimWs(encryptedData)))
      );
      return (new TextDecoder('utf-8')).decode(decrypted);
}
function str2ab (binaryStr) {
    const len = binaryStr.length;
    const bytes = new Uint8Array(len);
    for (let i = 0; i < len; i++) {
        bytes[i] = binaryStr.charCodeAt(i); // Accurately map each char to a byte
    }
    return bytes; // Convert to ArrayBuffer
};


(async () => {
  try {
    const privKey = await importPrivateKey(privKeyPEM);
    console.log(privKey);
    const decrypted = await decryptData(privKey, encryptedData);
  } catch (err) {
    console.error("Err:", err, "Err.massage:", err.message);
  }
})()

Solution

  • The OpenSSL rsautl encryption uses PKCS#1 v1.5 padding by default, while the WebCrypto code applies OAEP with SHA256 (which specifies SHA256 for both digests, the OAEP and the MGF1 digest). Therefore decryption fails.

    Since WebCrypto does not support PKCS#1 v1.5 padding, OAEP with SHA256 must be used on the OpenSSL side for decryption to be successful.

    OpenSSL rsautl supports OAEP, but only with SHA1. To use SHA256, OpenSSL pkeyutl must be applied. With the option -pkeyopt rsa_padding_mode:oaep OAEP is specified, with the options -pkeyopt rsa_oaep_md:sha256 -pkeyopt rsa_mgf1_md:sha256 the OAEP and the MGF1 digest are set to SHA256, altogether:

    echo -n "abcdef" | openssl pkeyutl -encrypt -pubin -inkey public_key.pem -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -pkeyopt rsa_mgf1_md:sha256 | openssl base64 > enc_simple.txt
    

    A ciphertext generated in this way can be decrypted with the posted WebCrypto code (for this test I used the private key from your posted WebCrypto code and extracted the public key for the encryption from it):

    const encryptedData = `ETL+N6TbCcU+qoSrtVHqamrcLBM2upvS04+rvjeSL3ZA370jDMDLMjzszEakVJ7E
    cA8mmY0GAK/ACFYV5Xc3KPXZVWFDaeTAu4tWY/TEftou/3zBWP9LjF4tViqWmxwC
    3zgXM6lFaRBJMU/hIdc2mjNNS8/Rvc4HIS+6WBPXK50AOPySuak6m1EU3Zp2xeny
    kGzH6+qotvJ3BC+OTXMAhaFwm+5hURY6rfAlYy5VX5NT2aD0FL0ggRF69Y+SMFZa
    DDmaTiV1OwHuErm1T1k26QTgGcX/W+MWIlFBDXbJZxbIKIaHqquK2mN7qrGEQKVz
    Ddh8MrfBs1aihAqgkXdLxg==`; //abcdef
    const privKeyPEM=`-----BEGIN PRIVATE KEY-----
    MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDJiQlvlYbTqi/W
    bLa7nzaF4Wz+ypQa01HD+UMHX25Uv3n6P69ucKs/OOdWbcenknAmSBK/VzgIsjlm
    vak3y1LoYImMhm8R26Arv8Fv7jwQkbBxVnOn6yfTL2h1SRK+zFzWumfbl2vp7POG
    s7CRQaK0HFPKroVlYHlinTlYXSlFqdaknH2ip1J2YPJ3O4w0/ME8gBZ0Mj1XkokB
    Qc0Lk76BpvfTuBbzvLYI4bq0/jSoM6ANecqK+T/7/aSFxh2x8acJ/xzJg2H7I+de
    pXZ1M9Mz0x94jWRfc9Q9qLgY6AuX7MUyb2YEs+vMxcim60fvktHAaXvn9vM0eauC
    B+hpThohAgMBAAECggEAINub0zqAwe/EXuRYopxhqlBHkf77SKhdc2MnX4NanKyf
    OYK6mnn6IZOoe/noDFUevc8QZ2vT8e8E1tBjT3px6PscUfH1F+dD5P4djp874cOv
    Dbt3ndAELTVUhZLFYKA3HrdDiZTVfk0oozSWvAgEe/MGYkwz6YRiJgbWO6bsvLOa
    5Ehhd7Ocxbs4B389ad3bVzRkk02/IRDEeO1z0+22Wv1NIe0Spzl6FVLspL6JdmEu
    60rpShfwN7D0G4TQYYp/9Sh+zt0ThIF4kqQoURx5E2uQaDUAT206itMG2mUWYSR1
    M37U+46kpWBz0jDaRFmqlySKrmkbx7kWXju8wP1ZAQKBgQDvlq4B3jAPd+Pu/Rxw
    GORV5S8PdJrkQnx8BeAviGAVc2w6EMyRWusVzL95mpRS0t0E5xZtB2kLkVU/I08b
    j+WqrGFqFp8niXYnJsien8nfLct0AhJVvVaPtt8lHP9SKniBOZc0owzFljnt+4/6
    M4ifAiKvJYOqPSdGpJgvJq4TsQKBgQDXVxJCjUNInb4p0WWljxdcMpgSDPn53+BG
    M/MrxVbTOvHhKsbNlglYxT/87J+8lFKS5Y9EkpqekCY1aGJ0PYfLBMowvNAwiP2s
    89++PTuEO/hK16Xb8znZ7rUGdJdRUitBN8iVkWNLA2pzalcZkkHbIWQbDuJOPbdO
    bo1u06U5cQKBgAgeliUQD5bmnD3kLAuMfGiAzNh8PieQLUHSvSc/OupfMALDwPsI
    FsF1X+PSHka0SLM61aK6RpASy83I94xakxD2qJJ808X2PZ/UC6Z8ic3bcnKrA04O
    jZlvPB643do+ADl45yvsfqlPjwUGqnlzN6UT4HMJFW42hlc5isLGT83xAoGBANT/
    UxhxEfRp0wcaECjKeJjBkpmILFp0jynhiM3qzA7zZv0JissfdO9RbBGJHBczvtl+
    J0/0kuv0OVbqgTfpBMBTZIsAuAzJ8+F2+AD8IDqT9uxQkcYVt0tRSc2w1VuioxZH
    TyhiPoycPFcdADpS6MEPLi11c3NgqEf0IgFVZ0CBAoGBAMybGC+IImSDpZzLaSpz
    KAD6Vyto7eK4pENH8bPcM2Hpy7aMGJkr6Q/NT34p/vSfFG2gm8ueILhvqQ8yB6h6
    gtfHOEu7CC/8WL4Y/WehrpD+fTTSOd/C+JZJbXjHjJ55NX1JzgeCPWJP2QOyXE85
    VLq+2V8bhsQxi4JylnH6suLB
    -----END PRIVATE KEY-----`
    const
        atob = (window || self)?.atob,
        subtle = (window || self).crypto.subtle,
        trimWs = (s) => s.replace(/\s*/g,""),
        trimHeader = (s) => s.replace(/-----(?:BEGIN|END) PRIVATE KEY-----/gi, "");
    async function importPrivateKey (pemkey) {
        const binaryKey = atob(trimWs(trimHeader(pemkey)));
        const keyBuffer = new Uint8Array(binaryKey.length);
        for (let i = 0; i < binaryKey.length; i++) {
            keyBuffer[i] = binaryKey.charCodeAt(i);
        }
        return await subtle.importKey(
            "pkcs8",
            keyBuffer,
            { name: "RSA-OAEP", hash: "SHA-256"},
            false,
            ["decrypt"]
        );
    }
    async function decryptData (privateKey, encryptedData) {
          const decrypted = await subtle.decrypt(
              { name: "RSA-OAEP", hash: "SHA-256"},
              privateKey,
             str2ab(atob(trimWs(encryptedData)))
          );
          return (new TextDecoder('utf-8')).decode(decrypted);
    }
    function str2ab (binaryStr) {
        const len = binaryStr.length;
        const bytes = new Uint8Array(len);
        for (let i = 0; i < len; i++) {
            bytes[i] = binaryStr.charCodeAt(i); // Accurately map each char to a byte
        }
        return bytes; // Convert to ArrayBuffer
    };
    
    
    (async () => {
      try {
        const privKey = await importPrivateKey(privKeyPEM);
        //console.log(privKey);
        const decrypted = await decryptData(privKey, encryptedData);
        console.log(decrypted)
      } catch (err) {
        console.error("Err:", err, "Err.massage:", err.message);
      }
    })()

    On the OpenSSL side it can be successfully decrypted with:

    base64 -d enc_simple.txt > enc_simple.bin
    openssl pkeyutl -decrypt -inkey private_key_pkcs8.pem -in enc_simple.bin -out decrypted_simple.txt -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -pkeyopt rsa_mgf1_md:sha256