javapdfhashpdfboxelectronic-signature

Java Apache PDFBox - issue with generating the hash for external signing or with merging it back - adding Public key certificate chain to PDF


Problem description

We have a specific business / technical case - we need to electronically sign PDFs, and to do that we're partnering with a company that handles electronic signature over the hash of a PDF.

We more or less know how to get the hash from the PDF and merge it back after signing by an external service via Rest API - but something along this process causes the PDF to show "Document has been altered or corrupted since it was signed".

Resulting PDF with broken signature

How we're doing it

We are using library Apache PDFBox version 3.0.0 (and we tried with different versions as well) and we're basing our solution on the official Apache Examples:

https://svn.apache.org/viewvc/pdfbox/trunk/examples/src/main/java/org/apache/pdfbox/examples/signature/

We've read many posts on stackoverlow that describe problems similar to ours, for example:

And we settled on the solution described here: Java PdfBox - PDF Sign Problem -External Signature -Invalid signature There are errors in formatting or in the information contained in this signature

Our main class - CreateSignature

package TOTPSignPDF.Service;

import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.interactive.digitalsignature.ExternalSigningSupport;
import org.apache.pdfbox.pdmodel.interactive.digitalsignature.PDSignature;
import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
import org.bouncycastle.cert.jcajce.JcaCertStore;
import org.bouncycastle.cms.CMSException;
import org.bouncycastle.cms.CMSSignedData;
import org.bouncycastle.cms.CMSSignedDataGenerator;
import org.bouncycastle.cms.CMSTypedData;
import org.bouncycastle.cms.jcajce.JcaSignerInfoGeneratorBuilder;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.operator.ContentSigner;
import org.bouncycastle.operator.DefaultSignatureAlgorithmIdentifierFinder;
import org.bouncycastle.operator.OperatorCreationException;
import org.bouncycastle.operator.jcajce.JcaDigestCalculatorProviderBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.security.GeneralSecurityException;
import java.security.MessageDigest;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.Arrays;
import java.util.Base64;
import java.util.Calendar;

/*
    This solution was based on official examples from Apache:
    https://svn.apache.org/viewvc/pdfbox/trunk/examples/src/main/java/org/apache/pdfbox/examples/signature/CreateSignature.java?revision=1899086&view=markup
    https://svn.apache.org/viewvc/pdfbox/trunk/examples/src/main/java/org/apache/pdfbox/examples/signature/CreateSignatureBase.java?view=markup
 */

@Service
public class CreateSignature {

    private static final Logger LOG = LoggerFactory.getLogger(CreateSignature.class);

    //@Autowired
    //ServerSignature serverSignature;

    @Autowired
    ExternalSignature externalSignature;

    private Certificate[] certificateChain;

    /**
     * Signs the given PDF file.
     *
     * @param inFile the original PDF file
     */
    public void signDocument(File inFile) throws IOException {

        // we're being given the certificate chain with public key
        setCertificateChain(externalSignature.getCertificateChain());

        String name = inFile.getName();
        String substring = name.substring(0, name.lastIndexOf('.'));

        File outFile = new File(inFile.getParent(), substring + "_signed.pdf");
        loadPDFAndSign(inFile, outFile);
    }

    private void setCertificateChain(final Certificate[] certificateChain) {
        this.certificateChain = certificateChain;
    }

    /**
     * Signs the given PDF file.
     *
     * @param inFile  input PDF file
     * @param outFile output PDF file
     * @throws IOException if the input file could not be read
     */

    private void loadPDFAndSign(File inFile, File outFile) throws IOException {
        if (inFile == null || !inFile.exists()) {
            throw new FileNotFoundException("Document for signing does not exist");
        }

        // sign
        try (FileOutputStream fileOutputStream = new FileOutputStream(outFile);
             PDDocument doc = Loader.loadPDF(inFile)) {
            addSignatureDictionaryAndSignExternally(doc, fileOutputStream);
        }
    }

    private void addSignatureDictionaryAndSignExternally(PDDocument document, OutputStream output)
        throws IOException {

        int accessPermissions = SigUtils.getMDPPermission(document);
        if (accessPermissions == 1) {
            throw new IllegalStateException("No changes to the document are permitted due to DocMDP transform parameters dictionary");
        }

        // create signature dictionary
        PDSignature signature = new PDSignature();
        signature.setFilter(PDSignature.FILTER_ADOBE_PPKLITE);
        signature.setSubFilter(PDSignature.SUBFILTER_ADBE_PKCS7_DETACHED);
        signature.setName("Example User");
        signature.setLocation("Los Angeles, CA");
        signature.setReason("Testing");

        // the signing date, needed for valid signature
        signature.setSignDate(Calendar.getInstance());

        // Optional: certify 
        if (accessPermissions == 0) {
            SigUtils.setMDPPermission(document, signature, 2);
        }

        // it was if(isExternalSigning()) {
        document.addSignature(signature);
        ExternalSigningSupport externalSigning = document.saveIncrementalForExternalSigning(output);
        // invoke external signature service - passing the document content with the "empty" signature added
        byte[] cmsSignature = sign(externalSigning.getContent());
        // set signature bytes received from the service
        externalSigning.setSignature(cmsSignature);
        //}

        // call SigUtils.checkCrossReferenceTable(document) if Adobe complains - we're doing it only because I ran out of ideas
        // and read https://stackoverflow.com/a/71293901/535646
        // and https://issues.apache.org/jira/browse/PDFBOX-5382
        SigUtils.checkCrossReferenceTable(document); // no errors here

        //document.close(); not needed because document is defined as auto-closeable resource in the function above
    }

    /**
     * SignatureInterface sample implementation.
     * Use your favorite cryptographic library to implement PKCS #7 signature creation.
     * If you want to create the hash and the signature separately (e.g. to transfer only the hash
     * to an external application), read <a href="https://stackoverflow.com/questions/41767351">this
     * answer</a> or <a href="https://stackoverflow.com/questions/56867465">this answer</a>.
     *
     * @throws IOException
     */
    private byte[] sign(InputStream content) throws IOException {
        try {

            // get the hash of the document with additional signature field
            MessageDigest digest = MessageDigest.getInstance("SHA256", new BouncyCastleProvider());

            byte[] hashBytes = digest.digest(content.readAllBytes());
            String hashBase64 = new String(Base64.getEncoder().encode(hashBytes));
            LOG.info("Digest in Base64: " + hashBase64);

            // call External API to sign the hash - hash of the document with added field for signature
            //byte[] signedHashBytes = serverSignature.sign(hashBase64);
            byte[] signedHashBytes = externalSignature.sign(hashBytes);

            // this lower part of the code is based on this answer:
            // https://stackoverflow.com/questions/69676156/java-pdfbox-pdf-sign-problem-external-signature-invalid-signature-there-are
            //
            // In the standalone application this ContentSigner would be an implementation of signing process,
            // but we are given the signed hash from the External Api and we just have to return it - the same situation
            // as in this stackoverflow post
            ContentSigner nonSigner_signedHashProvided = new ContentSigner() {

                @Override
                public byte[] getSignature() {
                    return signedHashBytes;
                }

                @Override
                public OutputStream getOutputStream() {
                    return new ByteArrayOutputStream();
                }

                @Override
                public AlgorithmIdentifier getAlgorithmIdentifier() {
                    return new DefaultSignatureAlgorithmIdentifierFinder().find("SHA256WithRSA");
                }
            };

            CMSSignedDataGenerator gen = new CMSSignedDataGenerator();
            X509Certificate certificateRepresentingTheSigner = (X509Certificate) certificateChain[0];

            gen.addSignerInfoGenerator(
                new JcaSignerInfoGeneratorBuilder(new JcaDigestCalculatorProviderBuilder().build())
                    .build(
                        nonSigner_signedHashProvided,
                        certificateRepresentingTheSigner
                    )
            );
            gen.addCertificates(new JcaCertStore(Arrays.asList(certificateChain)));

            //CMSProcessableInputStream msg = new CMSProcessableInputStream(content);
            //these don't seem to change anything
            //CMSTypedData msg = new CMSProcessableByteArray(signedHashBytes);
            CMSTypedData msg = new CMSProcessableInputStream(new ByteArrayInputStream("not used".getBytes()));
            CMSSignedData signedData = gen.generate(msg, false);

            return signedData.getEncoded();
        } catch (GeneralSecurityException | CMSException | OperatorCreationException e) {
            throw new IOException(e);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}

Class imitating the external hash signing process - ExternalSignature

I created a local keystore with self-signed private key with this command:

keytool -genkeypair -storepass 123456 -storetype pkcs12 -alias testKeystoreForPOC -validity 365 -keystore testKeystore.jks -keyalg RSA -sigalg SHA256withRSA

package TOTPSignPDF.Service;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.InvalidKeyException;
import java.security.KeyStore;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.PrivateKey;
import java.security.Signature;
import java.security.SignatureException;
import java.security.UnrecoverableKeyException;
import java.security.cert.Certificate;
import java.security.cert.CertificateEncodingException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Base64;
import java.util.Enumeration;

@Service
public class ExternalSignature {
    private static final Logger LOG = LoggerFactory.getLogger(ExternalSignature.class);

    private Certificate[] certificateChain;

    private PrivateKey privateKey;

    public byte[] sign(byte[] bytesToSign) {
        LOG.info("Started signing process");
        try {
            Signature privateSignature = Signature.getInstance("SHA256withRSA");
            privateSignature.initSign(privateKey);
            privateSignature.update(bytesToSign);
            byte[] signature = privateSignature.sign();
            LOG.info("Finished signing process");
            return signature;
        } catch (NoSuchAlgorithmException | InvalidKeyException | SignatureException e) {
            throw new RuntimeException(e);
        }
    }

    public Certificate[] getCertificateChain(){
        return this.certificateChain;
    }

    @PostConstruct
    private void initializePostConstruct() {
        File keystoreFile = new File("keystore/testKeystore.jks");

        try {
            KeyStore keystore = KeyStore.getInstance("PKCS12");

            char[] password = "123456".toCharArray();
            try (InputStream is = new FileInputStream(keystoreFile)) {
                keystore.load(is, password);
            }

            // grabs the first alias from the keystore and get the private key. An
            // alternative method or constructor could be used for setting a specific
            // alias that should be used.
            Enumeration<String> aliases = keystore.aliases();
            String alias;
            Certificate cert = null;
            while (cert == null && aliases.hasMoreElements()) {
                alias = aliases.nextElement();
                setPrivateKey((PrivateKey) keystore.getKey(alias, password));
                Certificate[] certChain = keystore.getCertificateChain(alias);
                if (certChain != null) {
                    setCertificateChain(certChain);
                    cert = certChain[0];
                    if (cert instanceof X509Certificate) {
                        // avoid expired certificate
                        ((X509Certificate) cert).checkValidity();

                        SigUtils.checkCertificateUsage((X509Certificate) cert);
                    }
                }
            }

            if (cert == null) {
                throw new IOException("Could not find certificate");
            }
        } catch (KeyStoreException | NoSuchAlgorithmException | CertificateException | UnrecoverableKeyException | IOException e) {
            throw new RuntimeException(e);
        }
    }

    public final void setPrivateKey(PrivateKey privateKey) {
        LOG.info("Set new private key");
        String base64OfPrivateKey = new String(Base64.getEncoder().encode(privateKey.getEncoded()));
        LOG.info("base64OfPrivateKey: " + base64OfPrivateKey);
        this.privateKey = privateKey;
    }

    public final void setCertificateChain(final Certificate[] certificateChain) {
        LOG.info("Set new certificate chain, size: " + certificateChain.length);
        try {
            String base64OfPublicKey = new String(Base64.getEncoder().encode(certificateChain[0].getEncoded()));
            LOG.info("base64OfPublicKey: " + base64OfPublicKey);
        } catch (CertificateEncodingException e) {
            throw new RuntimeException(e);
        }
        this.certificateChain = certificateChain;
    }
}

Additional classes that are used from Apache examples:

Please help us

As far as we understand other posts on this site revolving around similar problems, everything should work, but it doesn't. Most probably we're missing some crucial step.

We’re looking forward to any response, every bit of information could help us.

Something that I don't understand is why the digest (hash) of the PDF is so small: LXl/LHCaxrf7lYlN8d8m7gDNp9DRqY+azvxCS/mB3uY=

Does it look like correct size ?

PDF examples and public / private key


Solution

  • I found the solution to my problem in other Stackoverflow post: pdfbox - document getting corrupted after adding signed attributes

    Based on the answer from mkl, we were able to create a working solution with PDFBox version 3.0.0:

    package TOTPSignPDF.Service;
    
    import TOTPSignPDF.ServerSignature;
    import org.apache.pdfbox.Loader;
    import org.apache.pdfbox.pdmodel.PDDocument;
    import org.apache.pdfbox.pdmodel.interactive.digitalsignature.ExternalSigningSupport;
    import org.apache.pdfbox.pdmodel.interactive.digitalsignature.PDSignature;
    import org.apache.pdfbox.pdmodel.interactive.digitalsignature.SignatureOptions;
    import org.bouncycastle.asn1.ASN1EncodableVector;
    import org.bouncycastle.asn1.ASN1ObjectIdentifier;
    import org.bouncycastle.asn1.ASN1Primitive;
    import org.bouncycastle.asn1.DERSet;
    import org.bouncycastle.asn1.cms.Attribute;
    import org.bouncycastle.asn1.cms.AttributeTable;
    import org.bouncycastle.asn1.ess.ESSCertIDv2;
    import org.bouncycastle.asn1.ess.SigningCertificateV2;
    import org.bouncycastle.asn1.nist.NISTObjectIdentifiers;
    import org.bouncycastle.asn1.pkcs.PKCSObjectIdentifiers;
    import org.bouncycastle.asn1.x509.AlgorithmIdentifier;
    import org.bouncycastle.cert.X509CertificateHolder;
    import org.bouncycastle.cert.jcajce.JcaCertStore;
    import org.bouncycastle.cms.CMSAttributeTableGenerator;
    import org.bouncycastle.cms.CMSException;
    import org.bouncycastle.cms.CMSSignedData;
    import org.bouncycastle.cms.CMSSignedDataGenerator;
    import org.bouncycastle.cms.CMSTypedData;
    import org.bouncycastle.cms.DefaultSignedAttributeTableGenerator;
    import org.bouncycastle.cms.jcajce.JcaSignerInfoGeneratorBuilder;
    import org.bouncycastle.jcajce.io.OutputStreamFactory;
    import org.bouncycastle.operator.ContentSigner;
    import org.bouncycastle.operator.OperatorCreationException;
    import org.bouncycastle.operator.jcajce.JcaDigestCalculatorProviderBuilder;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import org.springframework.beans.factory.annotation.Autowired;
    import org.springframework.stereotype.Service;
    
    import java.io.File;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.io.OutputStream;
    import java.security.MessageDigest;
    import java.security.NoSuchAlgorithmException;
    import java.security.cert.Certificate;
    import java.security.cert.CertificateEncodingException;
    import java.security.cert.X509Certificate;
    import java.util.Arrays;
    import java.util.Base64;
    import java.util.Calendar;
    
    /*
        This solution was based on official examples from Apache:
        https://svn.apache.org/viewvc/pdfbox/trunk/examples/src/main/java/org/apache/pdfbox/examples/signature/CreateSignature.java?revision=1899086&view=markup
        https://svn.apache.org/viewvc/pdfbox/trunk/examples/src/main/java/org/apache/pdfbox/examples/signature/CreateSignatureBase.java?view=markup
     */
    
    @Service
    public class CreateSignature {
    
        private static final Logger LOG = LoggerFactory.getLogger(CreateSignature.class);
    
        @Autowired
        ServerSignature serverSignature;
    
        private Certificate[] certificateChain;
    
        /**
         * Signs the given PDF file.
         *
         * @param inFile the original PDF file
         */
        public void signDocument(File inFile, Certificate[] certChain) throws
                                                                       IOException,
                                                                       CertificateEncodingException,
                                                                       NoSuchAlgorithmException,
                                                                       OperatorCreationException,
                                                                       CMSException {
    
            // we're being given the certificate chain with public key
            setCertificateChain(certChain);
    
            String name = inFile.getName();
            String substring = name.substring(0, name.lastIndexOf('.'));
    
            File outFile = new File(inFile.getParent(), substring + "_signed_pdfbox.pdf");
            signDocument(inFile, outFile);
        }
    
        private void setCertificateChain(final Certificate[] certificateChain) {
            this.certificateChain = certificateChain;
        }
    
        // https://stackoverflow.com/questions/75505900/pdfbox-document-getting-corrupted-after-adding-signed-attributes
        private void signDocument(File inFile, File outFile) throws
                                                             IOException,
                                                             NoSuchAlgorithmException,
                                                             OperatorCreationException,
                                                             CertificateEncodingException,
                                                             CMSException {
            try (
                FileOutputStream output = new FileOutputStream(outFile);
                PDDocument document = Loader.loadPDF(inFile)
            ) {
                PDSignature signature = new PDSignature();
    
                signature.setFilter(PDSignature.FILTER_ADOBE_PPKLITE);
                signature.setSubFilter(PDSignature.SUBFILTER_ETSI_CADES_DETACHED);
                signature.setName("Test Name");
                signature.setSignDate(Calendar.getInstance());
    
                SignatureOptions signatureOptions = new SignatureOptions();
                signatureOptions.setPage(0);
    
                document.addSignature(signature, signatureOptions);
                ExternalSigningSupport externalSigning = document.saveIncrementalForExternalSigning(output);
    
                // retrieve signer certificate and its chain
                X509Certificate cert = (X509Certificate) certificateChain[0];
    
                // build signed attribute table generator and SignerInfo generator builder
                ESSCertIDv2 certid = new ESSCertIDv2(
                    new AlgorithmIdentifier(NISTObjectIdentifiers.id_sha256),
                    MessageDigest.getInstance("SHA-256").digest(cert.getEncoded())
                );
                SigningCertificateV2 sigcert = new SigningCertificateV2(certid);
                Attribute attr = new Attribute(PKCSObjectIdentifiers.id_aa_signingCertificateV2, new DERSet(sigcert));
    
                ASN1EncodableVector v = new ASN1EncodableVector();
                v.add(attr);
                AttributeTable atttributeTable = new AttributeTable(v);
                CMSAttributeTableGenerator attrGen = new DefaultSignedAttributeTableGenerator(atttributeTable);
    
                org.bouncycastle.asn1.x509.Certificate cert2 = org.bouncycastle.asn1.x509.Certificate.getInstance(ASN1Primitive.fromByteArray(cert.getEncoded()));
                JcaSignerInfoGeneratorBuilder sigb = new JcaSignerInfoGeneratorBuilder(new JcaDigestCalculatorProviderBuilder().build());
                sigb.setSignedAttributeGenerator(attrGen);
    
                // create ContentSigner that signs by calling the external endpoint
                ContentSigner contentSigner = new ContentSigner() {
                    private MessageDigest digest = MessageDigest.getInstance("SHA-256");
                    private OutputStream stream = OutputStreamFactory.createStream(digest);
    
                    @Override
                    public byte[] getSignature() {
                        try {
                            byte[] hash = digest.digest();
                            byte[] signedHash = serverSignature.sign(Base64.getEncoder().encodeToString(hash));
                            return signedHash;
                        } catch (Exception e) {
                            throw new RuntimeException("Exception while signing", e);
                        }
                    }
    
                    @Override
                    public OutputStream getOutputStream() {
                        return stream;
                    }
    
                    @Override
                    public AlgorithmIdentifier getAlgorithmIdentifier() {
                        return new AlgorithmIdentifier(new ASN1ObjectIdentifier("1.2.840.113549.1.1.11"));
                    }
                };
    
                // create the SignedData generator and execute
                CMSSignedDataGenerator gen = new CMSSignedDataGenerator();
                gen.addCertificates(new JcaCertStore(Arrays.asList(certificateChain)));
                gen.addSignerInfoGenerator(sigb.build(contentSigner, new X509CertificateHolder(cert2)));
    
                CMSTypedData msg = new CMSProcessableInputStream(externalSigning.getContent());
                CMSSignedData signedData = gen.generate(msg, false);
    
                byte[] cmsSignature = signedData.getEncoded();
                externalSigning.setSignature(cmsSignature);
            }
        }
    }