python python-3.x docker testcontainers podman

testcontainers initializing dockerclient twice - the second time on container.start

I am trying to create a container with oracle db to run tests. Due to some restrictions, I have to use rootless podman instead of docker. Here is how I do it:

def _container_env_kwargs() -> dict:
    # Try Podman rootless unix socket
    rootless = f"/var/run/user/{os.getuid()}/podman/podman.sock"
    if os.path.exists(rootless):
        try:
            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            s.settimeout(1.0)
            s.connect(rootless); s.close()
            env["DOCKER_HOST"] = f"unix://{rootless}"
            env.setdefault("DOCKER_API_VERSION", "1.41")
            env.setdefault("TESTCONTAINERS_RYUK_DISABLED", "true")
            return {"environment": env}
        except Exception:
            raise RuntimeError("Failed to create env for container client")

@pytest.fixture(scope="session")
def oracle_container() -> Generator[dict, None, None]:
    """
    Start an Oracle XE container using Podman.
    Wait until it is ready to accept SQL connections.
    """
    dk_kwargs = _container_env_kwargs()

    container = (
        DockerContainer(ORACLE_IMAGE, docker_client_kw=dk_kwargs)
        .with_env("ORACLE_PASSWORD", ORACLE_PASSWORD)
        .with_env("ORACLE_DATABASE", ORACLE_SERVICE)
        .with_exposed_ports("1521/tcp")
    )

    container.start()

When I try to run the tests, I get this stack trace:

The above exception was the direct cause of the following exception:

    @pytest.fixture(scope="session")
    def oracle_container() -> Generator[dict, None, None]:
        """
        Start an Oracle XE container using Docker or Podman.
        Wait until it is ready to accept SQL connections.
        """
        dk_kwargs = _container_env_kwargs()  # <- returns {"environment": {...}}
    
        container = (
            DockerContainer(ORACLE_IMAGE, docker_client_kw=dk_kwargs)
            .with_env("ORACLE_PASSWORD", ORACLE_PASSWORD)
            .with_env("ORACLE_DATABASE", ORACLE_SERVICE)
            .with_exposed_ports("1521/tcp")
        )
    
>       container.start()

conftest.py:125: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/testcontainers/core/container.py:176: in start
    Reaper.get_instance()
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/testcontainers/core/container.py:320: in get_instance
    Reaper._instance = Reaper._create_instance()
                       ^^^^^^^^^^^^^^^^^^^^^^^^^
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/testcontainers/core/container.py:343: in _create_instance
    DockerContainer(c.ryuk_image)
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/testcontainers/core/container.py:85: in __init__
    self._docker = DockerClient(**(docker_client_kw or {}))
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/testcontainers/core/docker_client.py:73: in __init__
    self.client = docker.from_env(**kwargs)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/docker/client.py:94: in from_env
    return cls(
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/docker/client.py:45: in __init__
    self.api = APIClient(*args, **kwargs)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^
../../miniconda3/envs/scoring-service-v-2/lib/python3.12/site-packages/docker/api/client.py:207: in __init__
    self._version = self._retrieve_server_version()
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <docker.api.client.APIClient object at 0x7f6a5593cc20>

    def _retrieve_server_version(self):
        try:
            return self.version(api_version=False)["ApiVersion"]
        except KeyError as ke:
            raise DockerException(
                'Invalid response from docker daemon: key "ApiVersion"'
                ' is missing.'
            ) from ke
        except Exception as e:
>           raise DockerException(
                f'Error while fetching server API version: {e}'
            ) from e
E           docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', FileNotFoundError(2, 'No such file or directory'))

When I debug and go step buy step, I see that the DockerClient object is created twice. The first time when I create DockerContainer object. And the self._version = self._retrieve_server_version() runs withour errors when creating the APIClient object the first time.

But then, when I hit container.start(), my breakpoints trigger again as if new DockerClient object is being created, but without env that I provide and I get the exception.

Why does it happen?

Solution

From the code you have provided, it is not clear why you hit the container.start() twice. This can have several reasons beyond the code itself like using a parallel test setup with pytest-xdist or similar, some IDE issues with debugging or miniconda issues...

But the code you're showing seems also not complete or is your expectation that you can simply use container.start() in the pytest fixture and then everything works as expected? Nevertheless, I have created a small setup to look into your issue:

import os
from os import _Environ, environ as env
import socket
from typing import Generator

import pytest
from testcontainers.core.container import DockerContainer


ORACLE_SHA256 = "c2682a4216b0fe65537912c4a049c7e5c15a85bb7c94bb8137d5f2d2eef60603"
ORACLE_TAG = "latest"
ORACLE_IMAGE = f"gvenzl/oracle-xe:{ORACLE_TAG}@sha256:{ORACLE_SHA256}"
ORACLE_PASSWORD = "my_password_which_I_really_should_change"


def _container_env_kwargs() -> dict[str, _Environ[str]] | None:
    # Try Podman rootless unix socket
    rootless = f"/var/run/user/{os.getuid()}/podman/podman.sock"
    if os.path.exists(rootless):
        try:
            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            s.settimeout(1.0)
            s.connect(rootless); s.close()
            env["DOCKER_HOST"] = f"unix://{rootless}"
            env.setdefault("DOCKER_API_VERSION", "1.41")
            env.setdefault("TESTCONTAINERS_RYUK_DISABLED", "true")
            return {"environment": env}
        except Exception:
            raise RuntimeError("Failed to create env for container client")
    return None


@pytest.fixture(scope="session")
def oracle_container() -> Generator[DockerContainer, None, None]:
    """
    Start an Oracle XE container using Podman.
    Wait until it is ready to accept SQL connections.
    """
    dk_kwargs = _container_env_kwargs()

    container = (
        DockerContainer(ORACLE_IMAGE, docker_client_kw=dk_kwargs)
        .with_env("ORACLE_PASSWORD", ORACLE_PASSWORD)
        .with_exposed_ports("1521/tcp")
    )

    container.start()
    yield container  # <- You probably need something for tests to consume
    container.stop()


def test_oracle(oracle_container: DockerContainer) -> None:
    """Test consumer for yielded `DockerContainer` (not that meaningful)"""
    assert oracle_container.env["ORACLE_PASSWORD"] == ORACLE_PASSWORD


if __name__ == "__main__":
    return_code = pytest.main(["-x"])

By using this code, I do not encounter an issue. But I am not using miniconda and using Linux and Python3.12 as well (3.12.7 to be more detailed), together with pytest 8.4.2 and testcontainers 4.13.2 in a poetry setup. So I can only assume that it has something to do with your environment or you have more code/dependencies running, which is not shown up here. This information is hopefully helpful.

I also looked into your code and have some other recommendations (formulated as Q&A), which may or may not help you to circumvent this issue as well (at least something you can try out):

Is there a special reason why you are using a generic DockerContainer?
- By using the OracleDbContainer (taken from this list) you get some convenience functions like get_connection_url() and others, which simplifies your code.
Why are you not using a yield statement (or do you have omitted something)?
- I can only assume that you have omitted some important code here. Generally, I would really recommend using context managers for this job. Then, you don't have to care about starting and stopping the container, it is done under the hood for you. By using the session-wide scope, the database is only created once per test session. Inside the context manager you can yield the (running) container. Only after all tests are done, the testcontainer will be stopped by leaving the with-statement body part.
Do you already have tests consuming the pytest fixture?
- Otherwise it is not clear how you get into the code. So you must have a consumer or the pytest fixture is not working as expected. I have added a very simple test to get the connection_url from the container. What I normally do, is to yield this connection_url instead of the container and then I can run my queries in the tests against this connection.

By saying this, you can refactor the code like this:

import os
from os import _Environ, environ as env
import socket
from typing import Generator

import pytest
from testcontainers.core.waiting_utils import wait_for_logs
from testcontainers.oracle import OracleDbContainer

ORACLE_SHA256 = "c2682a4216b0fe65537912c4a049c7e5c15a85bb7c94bb8137d5f2d2eef60603"
ORACLE_TAG = "latest"
ORACLE_IMAGE = f"gvenzl/oracle-xe:{ORACLE_TAG}@sha256:{ORACLE_SHA256}"
ORACLE_PASSWORD = "my_password_which_I_really_should_change"


def _container_env_kwargs() -> dict[str, _Environ[str]] | None:
    # Try Podman rootless unix socket
    rootless = f"/var/run/user/{os.getuid()}/podman/podman.sock"
    if os.path.exists(rootless):
        try:
            s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            s.settimeout(1.0)
            s.connect(rootless); s.close()
            env["DOCKER_HOST"] = f"unix://{rootless}"
            env.setdefault("DOCKER_API_VERSION", "1.41")
            env.setdefault("TESTCONTAINERS_RYUK_DISABLED", "true")
            return {"environment": env}
        except Exception:
            raise RuntimeError("Failed to create env for container client")
    return None


@pytest.fixture(scope="session")
def oracle_container() -> Generator[OracleDbContainer, None, None]:
    """
    Start an Oracle XE container using Podman.
    Wait until it is ready to accept SQL connections.
    """
    dk_kwargs = _container_env_kwargs()

    with (  # <- Using context manager
        OracleDbContainer(ORACLE_IMAGE, docker_client_kw=dk_kwargs)
        .with_env("ORACLE_PASSWORD", ORACLE_PASSWORD)
        .with_exposed_ports("1521/tcp")
    ) as oracle_container:

        # wait for container up and running
        wait_for_logs(oracle_container, "ORACLE instance started.")

        # yield the (running, healthy) container
        yield oracle_container

        # no special treatment after all tests are finished, simply
        # leave the context manager and container will be stopped


def test_oracle_has_connection_url(oracle_container: OracleDbContainer) -> None:
    assert oracle_container.get_connection_url().startswith("oracle+oracledb://system:")


if __name__ == "__main__":
    return_code = pytest.main(["-x"])

Notes:

wait_for_logs(oracle_container, "ORACLE instance started.") is one quick way to wait until the container is up and running. You should see such a message in the logs, of course. This can change over time.
I have left out .with_env("ORACLE_DATABASE", ORACLE_SERVICE) for simplicity, not needed for the small example.
Both docker and podman are installed on my machine. I have seen that podman is really creating the image and starting the container. Docker was not involved.