pythonjsonfilelogginglog-rotation

File rotation and deletion on json files


How can I implement fileRotation on json files. I know there exists a fileRotation handler in logging module of python but that is used to rotate and delete log files. My program is reading and writing using json module, although I can also use logging module to write in json form into the files but I don't think that is a good practice. Is there any other library I can use or will I have to write my own script to do rotation and deletion?


Solution

  • You can use the logging and json modules to achieve this as follows:

    import logging
    import json
    import os
    from logging.handlers import RotatingFileHandler
    
    def rotate_json_file(file_path, max_bytes, backup_count):
        logger = logging.getLogger(__name__)
        logger.setLevel(logging.INFO)
        
        file_handler = RotatingFileHandler(file_path, mode='a', maxBytes=max_bytes, backupCount=backup_count)
        file_handler.setFormatter(logging.Formatter('%(message)s'))
    
        logger.addHandler(file_handler)
        
        json_data = {'key': 'value'}
        logger.info(json.dumps(json_data))
    
        logger.removeHandler(file_handler)
        file_handler.close()
    
        record = logging.makeLogRecord(json_data)
        
        if file_handler.shouldRollover(record):
            # Get the rotated file names
            rotated_files = [file_path] + [file_path + '.{}'.format(i) for i in range(1, backup_count + 1)]
            
            os.remove(rotated_files[-1])
    
            for i in range(len(rotated_files) - 1, 0, -1):
                os.rename(rotated_files[i - 1], rotated_files[i])
    
            open(file_path, 'a').close()
    

    If you would like it to be Time based:

    import logging
    import json
    import os
    from logging.handlers import TimedRotatingFileHandler
    
    def rotate_json_file(file_path, when, interval, backup_count):
        logger = logging.getLogger(__name__)
        logger.setLevel(logging.INFO)
        
        file_handler = TimedRotatingFileHandler(file_path, when=when, interval=interval, backupCount=backup_count)
        file_handler.setFormatter(logging.Formatter('%(message)s'))
    
        logger.addHandler(file_handler)
        
        json_data = {'key': 'value'}
        logger.info(json.dumps(json_data))
    
        logger.removeHandler(file_handler)
        file_handler.close()
    
        record = logging.makeLogRecord(json_data)
        
        if file_handler.shouldRollover(record):
            # Get the rotated file names
            rotated_files = [file_handler.baseFilename] + file_handler.getFilesToDelete()
    
            os.remove(rotated_files[-1])
    
            for i in range(len(rotated_files) - 1, 0, -1):
                os.rename(rotated_files[i - 1], rotated_files[i])
    
            open(file_handler.baseFilename, 'a').close()
    

    Where we use the "TimedRotatingFileHandler" instead.