zip log files after it reach 10 backup count python

I am trying to zip logs once it reaches 10 backup counts in logging handler. Each log file am checking file size and am creating new file. Now if 10 files are getting created, for next file it should zip all logs and delete all 10 log files and start next new file till 10 files again.

Here is what I tried:

class CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler):
    def doRollover(self):
        """
        Do a rollover, as described in __init__().
        """
        
        if self.backupCount > 0:
            for i in range(self.backupCount - 1, 0, -1):
                sfn = "%s.%d.gz" % (self.baseFilename, i)
                dfn = "%s.%d.gz" % (self.baseFilename, i + 1)
                if os.path.exists(sfn):
                    # print "%s -> %s" % (sfn, dfn)
                    if os.path.exists(dfn):
                        os.remove(dfn)
                    os.rename(sfn, dfn)
            dfn = self.baseFilename + ".1.gz"
            if os.path.exists(dfn):
                os.remove(dfn)
            # These two lines below are the only new lines. I commented out the os.rename(self.baseFilename, dfn) and
            #  replaced it with these two lines.
            with open(self.baseFilename, 'rb') as f_in, gzip.open(dfn, 'wb') as f_out:
                shutil.copyfileobj(f_in, f_out)
            # os.rename(self.baseFilename, dfn)
            # print "%s -> %s" % (self.baseFilename, dfn)
        self.mode = 'w'

if __name__ == '__main__':
    name = 'test.log'
    logging.basicConfig(level=logging.INFO, filename=name, filemode="a+",
                format="%(asctime)-15s %(levelname)-8s %(message)s")
    log = logging.getLogger()
    handler = CompressedRotatingFileHandler(name, maxBytes=2000, backupCount=10)
    ii = 0
    while True:
        logging.debug("Test {0}".format(ii))
        ii += 1
        

But, I am not getting what I am expecting.

When you do the rollover, you need to create a .zip file with all the test.log.XX, delete them. In case you don't have 10 test.log.XX files, you should do the normal behavior of creating them.

The doRollover method looks like this:

import zipfile
import glob
import os

def doRollover(self):
    """
    Do a rollover, as described in __init__().
    """

    # This creates the .zip file from the 10 backup files
    
    # Count all the test.log.XXX that already exist
    log_files = [
        f"{self.baseFilename}.{i}"
        for i in range(1, self.backupCount + 1)
        if os.path.exists(f"{self.baseFilename}.{i}")
    ]

    # Count all the backup.XXX.zip files that already exist
    # to avoid overwriting one.
    nzip_backups = len(glob.glob('backup.*.zip'))

    # If there are 10 (self.backupCount) test.log.XXX files, create the zip and delete the files
    if len(log_files) == self.backupCount:
        with zipfile.ZipFile(f"backup.{nzip_backups + 1}.zip", "w") as myzip:
            for fn in log_files:
                if os.path.exists(fn):
                    myzip.write(fn, fn.split("/")[-1])
                    os.remove(fn)
    
    # In all cases, resort to the default behavior
    # (that copies test.log.4 -> test.log.5 etc.). Don't rewrite it,
    # just reuse the parent class' method.
    super().doRollover()

I also encourage you not to use the root logger (that you are using with log = logging.getLogger() and logging.debug("XXX")), but to use a child logger. Here, we will be using the "foobar" logger with logging.getLogger("foobar"). Since it does not already exist, we give it your handler and the correct Formatter.

By piecing everything together, it gives:

import logging
import logging.handlers
import os
import time
import glob
import zipfile


class CompressedRotatingFileHandler(logging.handlers.RotatingFileHandler):

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def doRollover(self):
        """
        Do a rollover, as described in __init__().
        """

        # This creates the .zip file from the 10 backup files

        log_files = [
            f"{self.baseFilename}.{i}"
            for i in range(1, self.backupCount + 1)
            if os.path.exists(f"{self.baseFilename}.{i}")
        ]

        # get count of backup.XXX.zip
        nzip_backups = len(glob.glob('backup.*.zip'))

        if len(log_files) == self.backupCount:
            with zipfile.ZipFile(f"backup.{nzip_backups + 1}.zip", "w") as myzip:
                for fn in log_files:
                    if os.path.exists(fn):
                        myzip.write(fn, fn.split("/")[-1])
                        os.remove(fn)
                        time.sleep(1)

        super().doRollover()


if __name__ == '__main__':
    name = 'test.log'
    log = logging.getLogger("foobar")
    log.setLevel(logging.DEBUG)
    log.propagate = False
    handler = CompressedRotatingFileHandler(
        name, mode="a+", maxBytes=2000, backupCount=10
    )
    handler.setFormatter(
        logging.Formatter(
            "%(asctime)-15s %(levelname)-8s %(message)s"
        )
    )
    log.addHandler(handler)

    ii = 0
    while True:
        log.info(f"Test {ii}")
        time.sleep(0.3)
        ii += 1