pythonloggingstdout

Python: redirect_stdout doesn't catch logger


I'm trying to redirect some logging output.

import sys
import logging
from contextlib import redirect_stdout

logging.basicConfig(stream=sys.stdout)
logger = logging.getLogger()

with open("out.txt", "w") as f:
    with redirect_stdout(f):
        print("STDOUT")
        logger.warning("LOGGER")

I would expect both of these to output to out.txt but instead STDOUT goes to the file, and WARNING:root:LOGGER displays in the terminal.

I've confirmed that that logger is outputting to stdout by removing the redirect stuff and running python script.py > stdout.txt. Then everything gets to the file, as it should.

So the logger must be doing something weird that causes it to not get pick up by redirect_stdout?

(Note: The behaviour is the same with redirect_stderr).


Solution

  • This is a solution/workaround that defines a custom function that replaces stdout and stderr with a different handler like the original redirect_stdout() and redirect_stderr() functions. However, it also adds an extra StreamHandler to the logger with the same target. This way the stdout, stderr and logger all write to the same stream. In the __exit__ method we reset the original streams, so this behaviour only applies to the code within the with block.

    import logging
    import sys
    from types import TracebackType
    from typing import TextIO
    
    
    class redirect_std:
        """Context manager to redirect stdout and stderr to a new target.
    
        This is an alternative implementation to the function from
        contextlib, which also adds a log handler to the root logger to
        capture log messages.
    
        Args:
            new_target: The new target to redirect stdout and stderr to.
        """
    
        def __init__(self, new_target: TextIO) -> None:
            self._new_target = new_target
            self._old_stdout: TextIO | None = None
            self._old_stderr: TextIO | None = None
            self._handler: logging.Handler | None = None
    
        def __enter__(self) -> TextIO:
            # Store the original stdout and stderr
            self._old_stdout = getattr(sys, "stdout")
            self._old_stderr = getattr(sys, "stderr")
    
            # Replace stdout and stderr with the new target
            setattr(sys, "stdout", self._new_target)
            setattr(sys, "stderr", self._new_target)
    
            # Add a handler to the root logger to capture the log messages
            formatter = logging.getLogger().handlers[0].formatter
            handler = logging.StreamHandler(stream=self._new_target)
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)
    
            # Return the new target so the caller can use it
            return self._new_target
    
        def __exit__(
            self,
            type_: type[BaseException] | None,
            value: BaseException | None,
            traceback: TracebackType | None,
        ) -> None:
            # Restore the original stdout and stderr
            setattr(sys, "stdout", self._old_stdout)
            setattr(sys, "stderr", self._old_stderr)
    
            # Remove the handler from the root logger
            if self._handler is not None:
                logging.getLogger().removeHandler(self._handler)
    
    
    logging.basicConfig(stream=sys.stdout)
    logger = logging.getLogger()
    logger.setLevel(logging.DEBUG)
    
    with open("out.txt", "w") as f:
        with redirect_std(f):
            print("STDOUT")
            logger.debug("DEBUG")
            logger.info("INFO")
            logger.warning("LOGGER")
            logger.error("ERROR")