I can't figure out why these two pieces of code generate different signatures when using the same message and key as inputs. Does anyone know what I'm doing wrong?

I've tried putting \0 or \n at the end of either message.

I've tried using ASCII encoding instead of UTF-8 for Apps Script.

I've tried different salt lengths for crypto.subtle (but I think Apps Script must use salt length of 0, because it generates the same signature every time.)

Google AppsScript

function testSign() {

  const message = "test";

  const key = `-----BEGIN PRIVATE KEY-----
MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDfIp9fsgiWRBWfieygtQaC0glxm7B5TbJAIDRMe6V9AJnMfSQ9nYpej+P5xhRIlXGT/WCtI3o9l/mIkh4Q8wmJg/HOZEoOyqPq2xFHLc9lksp2GWaQjFed0AUG+6KOJEAk6GHa6iyd0QwhN6bynzlS6Qd8i3fo3OrB9z+4YUrp4WWprswi5ogc91ckopTZvOJR1R+skm7dJ8amVkD7+OMufOGQNMYumgzsUJAQolCzxzlvz7D1L80gQRHF6IAMy4VaQezi/gB/xOVrMZFPD1Rk/lGDV1UlQVn0oxDluuGM1XCdKEcecQfMP77r0RZ5Tu6AtI4MuMSYTzliyrpe8n+4uYfBxhpi2aMLIusiUGTfYlCMKpWVi05wq6t2IgKXAQQEEWWYVWYH5CNwocHBPFn1wKJxt+qkKJsz5K4DtvAGg09f9xnRPZyMK3ZJK0W4Shuhj+MtujAh3/g2VTfOkiy8PUlcJD/4kdv/jKudDhlJ866whBVK2zgufWR4n6xfuq0RRbYagAQCAZwMNkkYi9g0yWkkw7U3sfSYiNuIZPTCtTan37gqnCfDYPVD7XjNKaZ+1UOggFwkun0+qcezfl9+k8wSEAkSbjEPfUeTwuIvYYPYLNvMbsd1E2fpTO5pGknDmE5P2JJqcufkz3iXCunOw/BScE6UxuDMFMX05OL0XQIDAQABAoICAA8B1V26i9zNYfXrsRRC8JfqV/PB4J3b9aXd9J6DP2dXm8B21zHkr1p6S38wTQtvr6agJzkl1nIfj0sZ5rdFnUnYK7JxqNBzXRBt4OzcXiNK+t50CWOl6LumsrvcPzvXoM/KqFAwqUUI+wud4lbVkiWrIhOEjtFE0G4wuqKkOoVd4ThHFxgu0I9ALGZ8n83AKCmQT7PL2nR52SC1UuQPgnoNMJ+CCSU3u7BGH9ZakFpzBwAn5BMtfpqRfchFgZ06r/KY1f2TT0XIoJRzzj3WvlqXhzRx8npRuaLcN8X3qnVOIqeTPqtDt15LmEPkeWI5xo0194rP/3rt8yJbl6rMnP3XolkdKtxogVpjrr+ywbvacSHyFKj0hiCrPjoXQXJR7l1hEsvtA0D74quFUnuS6dyZkk6AsWxXE0Yk2SelnYUtvdiGUtuv72vqepFilCMRcVuh0HuBAhtJ9VsSSnDm009aREvRs4svwbYuP/ryKkDCIkMbnzQII1H42JJGiWd7IvgZFzqv3wM1OCTisaMzHsC/bcr4Kb12i+NQLtp23C0V4GTRM++5CGLuDiQRSF9Q8OGeccIGdbM/kjIND6G1R3wQUoXZ61Mw7rOOctYsOdZGnf9Ghl4kxaS/eg9yjkzmm5dzK0Vw6o6ipFX7dhEqoZj92f6kgwuLX8HOW9AnJQGxAoIBAQD4UV5dba9FXGR7wfdx37tIg/HxCDRxcT5wZC7h7TcoFJsCJF5gnSMdGWsUNwFerZ74YnS9TRV3VJjn94KEgT+NGpAVhKnNqHPjWbxuOXjl7zMV3HaUJDrvzs+2e6s/uMXIEx4adf6YUowIkCGHjwYEM3hF48vg88unqD8lUSBPsEKxj6CJGq+NKi/eh2bRp2DFb2gBoqSHEywU11CZ35PMUipOWm419dv7qQn62+6UOiv0DjxWJRNRxt3R/meBnv/SGsKSIfl13H1QQg+aSW9xaDwwet/neDWYl20M2pBUPWYBIbpaz5ni/0WZOceaSbxh2pXZJi/L/B4tY33aVl7xAoIBAQDmCc/5Hq2a0uPd4NJifsZhF7hmNZNUQhkiWpW7NbwdI4w69sS714Pvs0zF+wXx43wWcec6a4yLK0Qqay0CcwdqMI6IKVfjPCpohVVHTgsOmQHBnfrEq3VUsGA8wQjWPNW9d/u9mLtGZYSm2BWaIWDSdHQzx28SsrsTu4yvYxoZdhenXsD9a1QYvu0X+C3UCfOdYC5VzNfUc86ELpQRunOhF8rhbbZNectCcijgc69yz5BLXtyfqvfWmiUBkFz+yb0o5RfEk0SSo60LH7P7wjuirAfASXqkGa7XGXAIRxq+JmHB0hqr1SoyQdMpSm7t5ug8IXgV1TTwP0mUS4ImwIQtAoIBABR7scsHJzQTgP5sa5rrF6nNqIF3acwJyVrACNX+GVSnDnpIwbg6fhECbcDHIMfMjpZymKqc1y52vf40foGrn7BmBoif3tnmEVkpp8930i81YgNloipqKqppZtzoqqGg/j+YxBzuqsep139FVF64P4jNLhilx8WQlrYHvN25KW8pXPcEn/tvRhfg6P30MVkN83+VxwCLiALUZAh8Elv/A1QRWwgHkZvF4hWKRhZ5Wd7ERafmHLgGaueN/fI6iBM7KGMObgpb3xYH0BZ0vJC/if/S11QwbpPLaLBjnU04IjuwrN9fBt5CzbDZ2cXf7EUf2/g+banx6nNrIpof4NvH0CECggEAHgdL4b4ydVJwMmeFrxvTc5swFA+MUuRp+YUPpKeIDdm1FYFe/xJMA79JF1MEXKYQbbGiuIqPhx83v73L21T+s8rw4C9dbKlO8+Pr1OoIIXixtP+VW5TyNQLtHSEpsSWx1RDTiNVmJPNdlJYCg+M1i2NuQ9AV3L/+Eb5ayA5MuuQihFOnJ62aBbzuoEFiYhqGdZW3lrWtuur/G1wlMgc/ztiXQEQdFxH+CYdzdJFFZtxXfq88Z49e2OG4UPLyYMQe8DavmpaKzgWVsi0KRqP9Oufv/xbYbpF3tFZ6vGnjwMyr2CxAFQw3fOYA1ZQE1QNeb3MDBP6W8YGhbj1JGRvqZQKCAQA2pJwXlgGc2TmBkqQiu2udzO9kueNqmH47V9tccoS7pfAI7NUH/6MH5hYYc45P63A7LlPyBYcHyIbJBHCQ7s1x8bGy4+d3BnmmQ2bRGWPC5ONo2mwZL4b3hZwhNqT68P3WgwBHOuuTdJ+pMotGrqyKkkEOYQ61/0x8M1xVcoVPUMutIKXqoJ7GjcD8WYddralNr735hOT1oF/3bGGRcKL2WNZRuZpiZXeJj1pGYRHmW7Rdof4lcFwz0zhAVVNiF4/CaaQhgZNGWW6MyfTllqBQoX5IOXvH49jaqEq6QA+crbbd634az9g7C5qIwcbVSDrtgb28AM5IARjOvDbqcJn9
-----END PRIVATE KEY-----
`;

  const sig = Utilities.computeRsaSha256Signature(message, key, Utilities.Charset.UTF_8);

  console.log(sig.slice(0,4)); // [ 45, -76, -60, -59 ]

}

Crypto.subtle:

  const message = "test";

  const pemContents = `MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDfIp9fsgiWRBWfieygtQaC0glxm7B5TbJAIDRMe6V9AJnMfSQ9nYpej+P5xhRIlXGT/WCtI3o9l/mIkh4Q8wmJg/HOZEoOyqPq2xFHLc9lksp2GWaQjFed0AUG+6KOJEAk6GHa6iyd0QwhN6bynzlS6Qd8i3fo3OrB9z+4YUrp4WWprswi5ogc91ckopTZvOJR1R+skm7dJ8amVkD7+OMufOGQNMYumgzsUJAQolCzxzlvz7D1L80gQRHF6IAMy4VaQezi/gB/xOVrMZFPD1Rk/lGDV1UlQVn0oxDluuGM1XCdKEcecQfMP77r0RZ5Tu6AtI4MuMSYTzliyrpe8n+4uYfBxhpi2aMLIusiUGTfYlCMKpWVi05wq6t2IgKXAQQEEWWYVWYH5CNwocHBPFn1wKJxt+qkKJsz5K4DtvAGg09f9xnRPZyMK3ZJK0W4Shuhj+MtujAh3/g2VTfOkiy8PUlcJD/4kdv/jKudDhlJ866whBVK2zgufWR4n6xfuq0RRbYagAQCAZwMNkkYi9g0yWkkw7U3sfSYiNuIZPTCtTan37gqnCfDYPVD7XjNKaZ+1UOggFwkun0+qcezfl9+k8wSEAkSbjEPfUeTwuIvYYPYLNvMbsd1E2fpTO5pGknDmE5P2JJqcufkz3iXCunOw/BScE6UxuDMFMX05OL0XQIDAQABAoICAA8B1V26i9zNYfXrsRRC8JfqV/PB4J3b9aXd9J6DP2dXm8B21zHkr1p6S38wTQtvr6agJzkl1nIfj0sZ5rdFnUnYK7JxqNBzXRBt4OzcXiNK+t50CWOl6LumsrvcPzvXoM/KqFAwqUUI+wud4lbVkiWrIhOEjtFE0G4wuqKkOoVd4ThHFxgu0I9ALGZ8n83AKCmQT7PL2nR52SC1UuQPgnoNMJ+CCSU3u7BGH9ZakFpzBwAn5BMtfpqRfchFgZ06r/KY1f2TT0XIoJRzzj3WvlqXhzRx8npRuaLcN8X3qnVOIqeTPqtDt15LmEPkeWI5xo0194rP/3rt8yJbl6rMnP3XolkdKtxogVpjrr+ywbvacSHyFKj0hiCrPjoXQXJR7l1hEsvtA0D74quFUnuS6dyZkk6AsWxXE0Yk2SelnYUtvdiGUtuv72vqepFilCMRcVuh0HuBAhtJ9VsSSnDm009aREvRs4svwbYuP/ryKkDCIkMbnzQII1H42JJGiWd7IvgZFzqv3wM1OCTisaMzHsC/bcr4Kb12i+NQLtp23C0V4GTRM++5CGLuDiQRSF9Q8OGeccIGdbM/kjIND6G1R3wQUoXZ61Mw7rOOctYsOdZGnf9Ghl4kxaS/eg9yjkzmm5dzK0Vw6o6ipFX7dhEqoZj92f6kgwuLX8HOW9AnJQGxAoIBAQD4UV5dba9FXGR7wfdx37tIg/HxCDRxcT5wZC7h7TcoFJsCJF5gnSMdGWsUNwFerZ74YnS9TRV3VJjn94KEgT+NGpAVhKnNqHPjWbxuOXjl7zMV3HaUJDrvzs+2e6s/uMXIEx4adf6YUowIkCGHjwYEM3hF48vg88unqD8lUSBPsEKxj6CJGq+NKi/eh2bRp2DFb2gBoqSHEywU11CZ35PMUipOWm419dv7qQn62+6UOiv0DjxWJRNRxt3R/meBnv/SGsKSIfl13H1QQg+aSW9xaDwwet/neDWYl20M2pBUPWYBIbpaz5ni/0WZOceaSbxh2pXZJi/L/B4tY33aVl7xAoIBAQDmCc/5Hq2a0uPd4NJifsZhF7hmNZNUQhkiWpW7NbwdI4w69sS714Pvs0zF+wXx43wWcec6a4yLK0Qqay0CcwdqMI6IKVfjPCpohVVHTgsOmQHBnfrEq3VUsGA8wQjWPNW9d/u9mLtGZYSm2BWaIWDSdHQzx28SsrsTu4yvYxoZdhenXsD9a1QYvu0X+C3UCfOdYC5VzNfUc86ELpQRunOhF8rhbbZNectCcijgc69yz5BLXtyfqvfWmiUBkFz+yb0o5RfEk0SSo60LH7P7wjuirAfASXqkGa7XGXAIRxq+JmHB0hqr1SoyQdMpSm7t5ug8IXgV1TTwP0mUS4ImwIQtAoIBABR7scsHJzQTgP5sa5rrF6nNqIF3acwJyVrACNX+GVSnDnpIwbg6fhECbcDHIMfMjpZymKqc1y52vf40foGrn7BmBoif3tnmEVkpp8930i81YgNloipqKqppZtzoqqGg/j+YxBzuqsep139FVF64P4jNLhilx8WQlrYHvN25KW8pXPcEn/tvRhfg6P30MVkN83+VxwCLiALUZAh8Elv/A1QRWwgHkZvF4hWKRhZ5Wd7ERafmHLgGaueN/fI6iBM7KGMObgpb3xYH0BZ0vJC/if/S11QwbpPLaLBjnU04IjuwrN9fBt5CzbDZ2cXf7EUf2/g+banx6nNrIpof4NvH0CECggEAHgdL4b4ydVJwMmeFrxvTc5swFA+MUuRp+YUPpKeIDdm1FYFe/xJMA79JF1MEXKYQbbGiuIqPhx83v73L21T+s8rw4C9dbKlO8+Pr1OoIIXixtP+VW5TyNQLtHSEpsSWx1RDTiNVmJPNdlJYCg+M1i2NuQ9AV3L/+Eb5ayA5MuuQihFOnJ62aBbzuoEFiYhqGdZW3lrWtuur/G1wlMgc/ztiXQEQdFxH+CYdzdJFFZtxXfq88Z49e2OG4UPLyYMQe8DavmpaKzgWVsi0KRqP9Oufv/xbYbpF3tFZ6vGnjwMyr2CxAFQw3fOYA1ZQE1QNeb3MDBP6W8YGhbj1JGRvqZQKCAQA2pJwXlgGc2TmBkqQiu2udzO9kueNqmH47V9tccoS7pfAI7NUH/6MH5hYYc45P63A7LlPyBYcHyIbJBHCQ7s1x8bGy4+d3BnmmQ2bRGWPC5ONo2mwZL4b3hZwhNqT68P3WgwBHOuuTdJ+pMotGrqyKkkEOYQ61/0x8M1xVcoVPUMutIKXqoJ7GjcD8WYddralNr735hOT1oF/3bGGRcKL2WNZRuZpiZXeJj1pGYRHmW7Rdof4lcFwz0zhAVVNiF4/CaaQhgZNGWW6MyfTllqBQoX5IOXvH49jaqEq6QA+crbbd634az9g7C5qIwcbVSDrtgb28AM5IARjOvDbqcJn9`;

  function str2ab(str) {
    const buf = new ArrayBuffer(str.length);
    const bufView = new Uint8Array(buf);
    for (let i = 0, strLen = str.length; i < strLen; i++) {
      bufView[i] = str.charCodeAt(i);
    }
    return buf;
  }

  function importRsaKey(pemContents) {
    const binaryDerString = window.atob(pemContents);
    const binaryDer = str2ab(binaryDerString);
    return window.crypto.subtle.importKey(
      "pkcs8",
      binaryDer,
      {
        name: "RSA-PSS",
        hash: "SHA-256"
      },
      true,
      ["sign"]
    );
  }

  const key = await importRsaKey(pemContents);

  const sig = await window.crypto.subtle.sign(
    {
      name: "RSA-PSS",
      saltLength: 0,
    },
    key,
    (new TextEncoder()).encode(message)
  );

  const sigArray = new Int8Array(sig);

  console.log(sigArray[0], sigArray[1], sigArray[2], sigArray[3]); // -97 -106 92 29

Looks like the problem was the algorithm for crypto.subtle. Following code for crypto.subtle matches appsscript:

  const message = "test";

  const pemContents = `MIIJQQIBADANBgkqhkiG9w0BAQEFAASCCSswggknAgEAAoICAQDfIp9fsgiWRBWfieygtQaC0glxm7B5TbJAIDRMe6V9AJnMfSQ9nYpej+P5xhRIlXGT/WCtI3o9l/mIkh4Q8wmJg/HOZEoOyqPq2xFHLc9lksp2GWaQjFed0AUG+6KOJEAk6GHa6iyd0QwhN6bynzlS6Qd8i3fo3OrB9z+4YUrp4WWprswi5ogc91ckopTZvOJR1R+skm7dJ8amVkD7+OMufOGQNMYumgzsUJAQolCzxzlvz7D1L80gQRHF6IAMy4VaQezi/gB/xOVrMZFPD1Rk/lGDV1UlQVn0oxDluuGM1XCdKEcecQfMP77r0RZ5Tu6AtI4MuMSYTzliyrpe8n+4uYfBxhpi2aMLIusiUGTfYlCMKpWVi05wq6t2IgKXAQQEEWWYVWYH5CNwocHBPFn1wKJxt+qkKJsz5K4DtvAGg09f9xnRPZyMK3ZJK0W4Shuhj+MtujAh3/g2VTfOkiy8PUlcJD/4kdv/jKudDhlJ866whBVK2zgufWR4n6xfuq0RRbYagAQCAZwMNkkYi9g0yWkkw7U3sfSYiNuIZPTCtTan37gqnCfDYPVD7XjNKaZ+1UOggFwkun0+qcezfl9+k8wSEAkSbjEPfUeTwuIvYYPYLNvMbsd1E2fpTO5pGknDmE5P2JJqcufkz3iXCunOw/BScE6UxuDMFMX05OL0XQIDAQABAoICAA8B1V26i9zNYfXrsRRC8JfqV/PB4J3b9aXd9J6DP2dXm8B21zHkr1p6S38wTQtvr6agJzkl1nIfj0sZ5rdFnUnYK7JxqNBzXRBt4OzcXiNK+t50CWOl6LumsrvcPzvXoM/KqFAwqUUI+wud4lbVkiWrIhOEjtFE0G4wuqKkOoVd4ThHFxgu0I9ALGZ8n83AKCmQT7PL2nR52SC1UuQPgnoNMJ+CCSU3u7BGH9ZakFpzBwAn5BMtfpqRfchFgZ06r/KY1f2TT0XIoJRzzj3WvlqXhzRx8npRuaLcN8X3qnVOIqeTPqtDt15LmEPkeWI5xo0194rP/3rt8yJbl6rMnP3XolkdKtxogVpjrr+ywbvacSHyFKj0hiCrPjoXQXJR7l1hEsvtA0D74quFUnuS6dyZkk6AsWxXE0Yk2SelnYUtvdiGUtuv72vqepFilCMRcVuh0HuBAhtJ9VsSSnDm009aREvRs4svwbYuP/ryKkDCIkMbnzQII1H42JJGiWd7IvgZFzqv3wM1OCTisaMzHsC/bcr4Kb12i+NQLtp23C0V4GTRM++5CGLuDiQRSF9Q8OGeccIGdbM/kjIND6G1R3wQUoXZ61Mw7rOOctYsOdZGnf9Ghl4kxaS/eg9yjkzmm5dzK0Vw6o6ipFX7dhEqoZj92f6kgwuLX8HOW9AnJQGxAoIBAQD4UV5dba9FXGR7wfdx37tIg/HxCDRxcT5wZC7h7TcoFJsCJF5gnSMdGWsUNwFerZ74YnS9TRV3VJjn94KEgT+NGpAVhKnNqHPjWbxuOXjl7zMV3HaUJDrvzs+2e6s/uMXIEx4adf6YUowIkCGHjwYEM3hF48vg88unqD8lUSBPsEKxj6CJGq+NKi/eh2bRp2DFb2gBoqSHEywU11CZ35PMUipOWm419dv7qQn62+6UOiv0DjxWJRNRxt3R/meBnv/SGsKSIfl13H1QQg+aSW9xaDwwet/neDWYl20M2pBUPWYBIbpaz5ni/0WZOceaSbxh2pXZJi/L/B4tY33aVl7xAoIBAQDmCc/5Hq2a0uPd4NJifsZhF7hmNZNUQhkiWpW7NbwdI4w69sS714Pvs0zF+wXx43wWcec6a4yLK0Qqay0CcwdqMI6IKVfjPCpohVVHTgsOmQHBnfrEq3VUsGA8wQjWPNW9d/u9mLtGZYSm2BWaIWDSdHQzx28SsrsTu4yvYxoZdhenXsD9a1QYvu0X+C3UCfOdYC5VzNfUc86ELpQRunOhF8rhbbZNectCcijgc69yz5BLXtyfqvfWmiUBkFz+yb0o5RfEk0SSo60LH7P7wjuirAfASXqkGa7XGXAIRxq+JmHB0hqr1SoyQdMpSm7t5ug8IXgV1TTwP0mUS4ImwIQtAoIBABR7scsHJzQTgP5sa5rrF6nNqIF3acwJyVrACNX+GVSnDnpIwbg6fhECbcDHIMfMjpZymKqc1y52vf40foGrn7BmBoif3tnmEVkpp8930i81YgNloipqKqppZtzoqqGg/j+YxBzuqsep139FVF64P4jNLhilx8WQlrYHvN25KW8pXPcEn/tvRhfg6P30MVkN83+VxwCLiALUZAh8Elv/A1QRWwgHkZvF4hWKRhZ5Wd7ERafmHLgGaueN/fI6iBM7KGMObgpb3xYH0BZ0vJC/if/S11QwbpPLaLBjnU04IjuwrN9fBt5CzbDZ2cXf7EUf2/g+banx6nNrIpof4NvH0CECggEAHgdL4b4ydVJwMmeFrxvTc5swFA+MUuRp+YUPpKeIDdm1FYFe/xJMA79JF1MEXKYQbbGiuIqPhx83v73L21T+s8rw4C9dbKlO8+Pr1OoIIXixtP+VW5TyNQLtHSEpsSWx1RDTiNVmJPNdlJYCg+M1i2NuQ9AV3L/+Eb5ayA5MuuQihFOnJ62aBbzuoEFiYhqGdZW3lrWtuur/G1wlMgc/ztiXQEQdFxH+CYdzdJFFZtxXfq88Z49e2OG4UPLyYMQe8DavmpaKzgWVsi0KRqP9Oufv/xbYbpF3tFZ6vGnjwMyr2CxAFQw3fOYA1ZQE1QNeb3MDBP6W8YGhbj1JGRvqZQKCAQA2pJwXlgGc2TmBkqQiu2udzO9kueNqmH47V9tccoS7pfAI7NUH/6MH5hYYc45P63A7LlPyBYcHyIbJBHCQ7s1x8bGy4+d3BnmmQ2bRGWPC5ONo2mwZL4b3hZwhNqT68P3WgwBHOuuTdJ+pMotGrqyKkkEOYQ61/0x8M1xVcoVPUMutIKXqoJ7GjcD8WYddralNr735hOT1oF/3bGGRcKL2WNZRuZpiZXeJj1pGYRHmW7Rdof4lcFwz0zhAVVNiF4/CaaQhgZNGWW6MyfTllqBQoX5IOXvH49jaqEq6QA+crbbd634az9g7C5qIwcbVSDrtgb28AM5IARjOvDbqcJn9`;

  function str2ab(str) {
    const buf = new ArrayBuffer(str.length);
    const bufView = new Uint8Array(buf);
    for (let i = 0, strLen = str.length; i < strLen; i++) {
      bufView[i] = str.charCodeAt(i);
    }
    return buf;
  }

  function importRsaKey(pemContents) {
    const binaryDerString = window.atob(pemContents);
    const binaryDer = str2ab(binaryDerString);
    return window.crypto.subtle.importKey(
      "pkcs8",
      binaryDer,
      {
        name: "RSASSA-PKCS1-v1_5",
        hash: "SHA-256"
      },
      true,
      ["sign"]
    );
  }

  const key = await importRsaKey(pemContents);

  const sig = await window.crypto.subtle.sign(
    {
      name: "RSASSA-PKCS1-v1_5"
    },
    key,
    (new TextEncoder()).encode(message)
  );

  const sigArray = new Int8Array(sig);

  console.log(sigArray[0], sigArray[1], sigArray[2], sigArray[3]); // 45 -76 -60 -59