Purpose of this question

This question was created to share information, because the documentation for various modules is limited on this topic. This question might become a community wiki.

Use Case

How to securely move sensitive data from memory to disk?

The broader use case is how to do this without exposing the sensitive data by writing it to a temporary file on the local disk.

What is sensitive data?

Sensitive data is information that must be protected and is inaccessible to outside parties unless specifically granted permission to view the data. This type of data can include, but not limited to PII (Personally identifiable information), PHI (Protected health information) or GDPR sensitive data.

It some cases the sensitive data might be generated during an investigation of a computer system for legal reasons (e.g., surveillance warrant) or covert reasons (e.g. spying on a potential adversary).

There are several ways to accomplish this and I will be providing several examples that either use cleartext data or encrypted data and encrypted compressed files with password protection.

pyzipper

Here is one method to accomplish this use case using pyzipper

import pyzipper

# generate data
secret_data = b'xyz' * 10

# password for the zip file
secret_password = b'super secret password'

# Create encrypted password protected ZIP file on disk
with pyzipper.AESZipFile('password_protected.zip',
                         'w',
                         compression=pyzipper.ZIP_LZMA,
                         encryption=pyzipper.WZ_AES) as zf:
    zf.setpassword(secret_password)
    zf.writestr('secret_data.txt', secret_data)


# Read encrypted password protected ZIP file from disk
with pyzipper.AESZipFile('password_protected.zip', 'r') as zf:
    zf.setpassword(secret_password)
    my_secrets = zf.read('secret_data.txt')
    print(my_secrets)
    # output
    xyzxyzxyzxyzxyzxyzxyzxyzxyzxyz

When I was reviewing the code for pyzipper it wasn't clear if the secret data was being protected during the read and write processes. For extra security I decided to encrypt the secret data using the cryptography package. This package was first released in 2014 and has been continually maintained by hundreds of contributors.

import os
import base64
import pyzipper
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC

# password for the cryptographic process
password = b"password"

# random salt for the cryptographic process
salt = os.urandom(16)

# key derivation function (KDF) for the cryptographic process
kdf = PBKDF2HMAC(
    algorithm=hashes.SHA256(),
    length=32,
    salt=salt,
    iterations=390000,
)

# create the encryption key for the cryptographic process
encryption_key = base64.urlsafe_b64encode(kdf.derive(password))

# https://cryptography.io/en/latest/fernet/
f = Fernet(encryption_key)

# generate data
secret_data = b'xyz' * 10

encrypted_data = f.encrypt(secret_data)

# password for the zip file
secret_password = b'super secret password'

# Create encrypted password protected ZIP file on disk
with pyzipper.AESZipFile('password_protected.zip',
                         'w',
                         compression=pyzipper.ZIP_LZMA,
                         encryption=pyzipper.WZ_AES) as zf:
    zf.setpassword(secret_password)
    zf.writestr('encryption_key.txt', encryption_key)
    zf.writestr('encrypted.txt', encrypted_data)


# Read encrypted password protected ZIP file from disk
# and decrypt data
with pyzipper.PyZipFile('password_protected.zip', 'r') as zf:
    zf.setpassword(secret_password)
    my_secrets = zf.read('encrypted.txt')
    my_key = zf.read('encryption_key.txt')
    f2 = Fernet(my_key)
    decrypt_message = f2.decrypt(my_secrets)
    print(decrypt_message)
    # output
    b'xyzxyzxyzxyzxyzxyzxyzxyzxyzxyz'

For an added layer of security an in-memory file system can be used. In this scenario the secret data is encrypted and then written to an in-memory file, which is then written to an encrypted password protected ZIP file.

import fs
import os
import base64
import pyzipper
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC

# create in-memory file system
mem_fs = fs.open_fs('mem://')
mem_fs.makedir('hidden_dir')

# password for the cryptographic process
password = b"password"

# random salt for the cryptographic process
salt = os.urandom(16)

# key derivation function (KDF) for the cryptographic process
kdf = PBKDF2HMAC(
    algorithm=hashes.SHA256(),
    length=32,
    salt=salt,
    iterations=390000,
)

# create the encryption key for the cryptographic process
encryption_key = base64.urlsafe_b64encode(kdf.derive(password))

# https://cryptography.io/en/latest/fernet/
f = Fernet(encryption_key)

# generate data
secret_data = b'xyz' * 10

encrypted_data = f.encrypt(secret_data)

# password for the zip file
secret_password = b'super secret password'


# write encryption key to an in-memory file
with mem_fs.open('hidden_dir/encryption_key.txt', 'wb') as in_file_in_memory:
    in_file_in_memory.write(encryption_key)
    in_file_in_memory.close()

# write encrypted data to an in-memory file
with mem_fs.open('hidden_dir/encrypted.txt', 'wb') as in_file_in_memory:
    in_file_in_memory.write(encrypted_data)
    in_file_in_memory.close()


# Create encrypted password protected ZIP file on disk
with pyzipper.AESZipFile('password_protected.zip',
                         'w',
                         compression=pyzipper.ZIP_LZMA,
                         encryption=pyzipper.WZ_AES) as zf:
    zf.setpassword(secret_password)
    zf.writestr('encryption_key.txt', mem_fs.readbytes('hidden_dir/encryption_key.txt'))
    zf.writestr('encrypted.txt', mem_fs.readbytes('hidden_dir/encrypted.txt'))


# Read encrypted password protected ZIP file from disk
# and decrypt data
with pyzipper.PyZipFile('password_protected.zip', 'r') as zf:
    zf.setpassword(secret_password)
    my_secrets = zf.read('encrypted.txt')
    my_key = zf.read('encryption_key.txt')
    f2 = Fernet(my_key)
    decrypt_message = f2.decrypt(my_secrets)
    print(decrypt_message)
    b'xyzxyzxyzxyzxyzxyzxyzxyzxyzxyz'

7zip

Another way to accomplish the use case is by using 7zip and the Python subprocess module. For this example I show how to create the in-memory file system, encrypt the data, write the data and the encryption key to a directory in memory, write these files to an encrypted password protected 7zip file on the disk. I also showed how to extract the files from this 7zip archive and decrypt the secret data.

import fs
import os
import base64
from subprocess import Popen, PIPE
from cryptography.fernet import Fernet
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC

# create in-memory file system
mem_fs = fs.open_fs('mem://')
mem_fs.makedir('hidden_dir')

# password for the cryptographic process
password = b"password"

# random salt for the cryptographic process
salt = os.urandom(16)

# key derivation function (KDF) for the cryptographic process
kdf = PBKDF2HMAC(
    algorithm=hashes.SHA256(),
    length=32,
    salt=salt,
    iterations=390000,
)

# create the encryption key for the cryptographic process
encryption_key = base64.urlsafe_b64encode(kdf.derive(password))

# https://cryptography.io/en/latest/fernet/
f = Fernet(encryption_key)

# generate data
secret_data = b'xyz' * 10

encrypted_data = f.encrypt(secret_data)

# password for the zip file
secret_password = b'super secret password'

# write encryption key to an in-memory file
with mem_fs.open('hidden_dir/encryption_key.txt', 'wb') as in_file_in_memory:
    in_file_in_memory.write(encryption_key)
    in_file_in_memory.close()

# write encrypted data to an in-memory file
with mem_fs.open('hidden_dir/encrypted.txt', 'wb') as in_file_in_memory:
    in_file_in_memory.write(encrypted_data)
    in_file_in_memory.close()

# Create encrypted password protected 7ZIP file on disk
file_names = ['encryption_key.txt', 'encrypted.txt']
data_elements = [mem_fs.readbytes('hidden_dir/encryption_key.txt'), mem_fs.readbytes('hidden_dir/encrypted.txt')]
for file_name, data_element in zip(file_names, data_elements):
    args = [f'{full_path}/7zz', 'a', '-mem=AES256', '-y', f'-p{secret_password}', f'-si{file_name}', f'{zip_archive_name}']
    proc_zip = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
    proc_zip.stdin.write(data_element)
    proc_zip.communicate()

# Unzip encrypted password protected 7ZIP file from disk
outfile_directory_name = 'test_files'
args = [f'{full_path}/7zz', "x", f'-p{secret_password}', f"{zip_archive_name}", f'-o{outfile_directory_name}']
proc_unzip = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
proc_unzip.communicate()

# decrypt data from disk
with open(f'{full_path}/{outfile_directory_name}/encryption_key.txt', 'rb') as key_file:
    with open(f'{full_path}/{outfile_directory_name}/encrypted.txt', 'rb') as text_file:
        f = Fernet(key_file.read())
        print(f.decrypt(text_file.read()))
        # output
        b'xyzxyzxyzxyzxyzxyzxyzxyzxyzxyz'