pythonbinaries

Memory Error in Python when trying to convert large files to binary


I'm working on developing a program to generate a dictionary containing binaries of various files for a Minecraft modpack. Everything works smoothly when processing smaller files. However, when attempting to process larger files, such as the "Better End Reforked" mod which is 90MB in size, the program encounters a "Memory Error" and fails.

This is my code:

from os      import listdir
from os.path import abspath, splitext, basename, getsize, isfile, join, expandvars

def getBool(statement):
    return (True if statement else False)

def getFiles(path):
    list = []

    for file in listdir(path):
        filePath = join(path, file)
        if isfile(filePath):
            list.append((file, abspath(filePath)))
    
    return list

def getIdentity(path):
    name = basename(path)
    
    return {
        'name': name,
        'basename': splitext(name)[0], 
        'extension': splitext(name)[1],
        'realpath': abspath(expandvars(path)),
        'size': getsize(path)
    }

def fromFileToBinary(path):
    try:
        with open(path, 'rb') as file:
            binary = file.read()
            return binary.hex()
    except IOError:
        return None


output = getIdentity('./output/bin.py')['realpath']

config       = 'C:/Users/Berdy Alexei/Downloads/modpack/optional/config'
mod          = 'C:/Users/Berdy Alexei/Downloads/modpack/optional/mods'
resourcepack = 'C:/Users/Berdy Alexei/Downloads/modpack/optional/resourcepacks'
script       = 'C:/Users/Berdy Alexei/Downloads/modpack/optional/scripts'

bool = getBool(config or mod or resourcepack or script)

def _bin(path = None, bool = True):
    dictionary = {}

    if path and bool:
        files = getFiles(path)
        
        for file in files:
            name, path = file
            dictionary[name] = fromFileToBinary(path)

    return dictionary

content = {
    'default': {
        'config':       _bin('C:/Users/Berdy Alexei/Downloads/modpack/default/config', False),
        'mods':         _bin('C:/Users/Berdy Alexei/Downloads/modpack/default/mods', False),
        'resourcepack': _bin('C:/Users/Berdy Alexei/Downloads/modpack/default/resourcepacks', False),
        'script':       _bin('C:/Users/Berdy Alexei/Downloads/modpack/default/scripts', False)
    },
    'optional': (bool, {
        'config':       _bin(config),
        'mods':         _bin(mod),
        'resourcepack': _bin(resourcepack),
        'script':       _bin(script)
    })
}

with open(output, "w") as file:
    file.write('BIN = {}'.format(content))

This is the error:

Traceback (most recent call last):
  File "c:\Folders\Archivos\Proyectos\InstallerCrafter\InstallerCrafter.py", line 75, in <module>
    file.write('BIN = {}'.format(content))
MemoryError

These are the files I'm processing: enter image description here


Solution

  • My solution:

    def fromFileToBinary(path, size = 4):
        try:
            binary = ''
    
            with open(path, 'rb') as file:
                while True:
                    chunk = file.read(1024 *  size)
    
                    if not chunk:
                        break
    
                    binary += chunk.hex()
                    del chunk
    
            return binary
        except IOError:
            return None
    

    The disadvantage is that it takes a long time to generate the binary, however it can be improved by adding that, after a certain weight of the file, the function is executed in this way or not (See the "getIdentity" function above in the code).