pythoncyclic-dependency

Avoiding circular imports for the 100th time


Summary

I keep on having an ImportError in a complex project. I've distilled it to the bare minimum that still gives the error.

Example

A wizard has containers with green and brown potions. These can be added together, resulting in new potions that are also either green or brown.

We have a Potion ABC, which gets its __add__, __neg__ and __mul__ from the PotionArithmatic mixin. Potion has 2 subclasses: GreenPotion and BrownPotion.

In one file, it looks like this:

onefile.py:

from abc import ABC, abstractmethod

def add_potion_instances(potion1, potion2): # some 'outsourced' arithmatic
    return BrownPotion(potion1.volume + potion2.volume)

class PotionArithmatic:
    def __add__(self, other):
        # Adding potions always returns a brown potion.
        if isinstance(other, base.Potion):
            return add_potion_instances(self, other)
        return BrownPotion(self.volume + other)

    def __mul__(self, other):
        # Multiplying a potion with a number scales it.
        if isinstance(other, Potion):
            raise TypeError("Cannot multiply Potions")
        return self.__class__(self.volume * other)

    def __neg__(self):
        # Negating a potion changes its color but not its volume.
        if isinstance(self, GreenPotion):
            return BrownPotion(self.volume)
        else:  # isinstance(self, BrownPotion):
            return GreenPotion(self.volume)

    # (... and many more)


class Potion(ABC, PotionArithmatic):
    def __init__(self, volume: float):
        self.volume = volume

    __repr__ = lambda self: f"{self.__class__.__name__} with volume of {self.volume} l."

    @property
    @abstractmethod
    def color(self) -> str:
        ...


class GreenPotion(Potion):
    color = "green"


class BrownPotion(Potion):
    color = "brown"


if __name__ == "__main__":

    b1 = GreenPotion(5)
    b2 = BrownPotion(111)

    b3 = b1 + b2
    assert b3.volume == 116
    assert type(b3) is BrownPotion

    b4 = b1 * 3
    assert b4.volume == 15
    assert type(b4) is GreenPotion

    b5 = b2 * 3
    assert b5.volume == 333
    assert type(b5) is BrownPotion

    b6 = -b1
    assert b6.volume == 5
    assert type(b6) is BrownPotion

This works.

Split into files into importable module

Each part is put in its own file inside the folder potions, like so:

usage.py
potions
| arithmatic.py
| base.py
| green.py
| brown.py
| __init__.py

potions/arithmatic.py:

from . import base, brown, green

def add_potion_instances(potion1, potion2):
    return brown.BrownPotion(potion1.volume + potion2.volume)

class PotionArithmatic:
    def __add__(self, other):
        # Adding potions always returns a brown potion.
        if isinstance(other, base.Potion):
            return add_potion_instances(self, other)
        return brown.BrownPotion(self.volume + other)

    def __mul__(self, other):
        # Multiplying a potion with a number scales it.
        if isinstance(other, base.Potion):
            raise TypeError("Cannot multiply Potions")
        return self.__class__(self.volume * other)

    def __neg__(self):
        # Negating a potion changes its color but not its volume.
        if isinstance(self, green.GreenPotion):
            return brown.BrownPotion(self.volume)
        else:  # isinstance(self, BrownPotion):
            return green.GreenPotion(self.volume)

potions/base.py:

from abc import ABC, abstractmethod
from .arithmatic import PotionArithmatic

class Potion(ABC, PotionArithmatic):
    def __init__(self, volume: float):
        self.volume = volume

    __repr__ = lambda self: f"{self.__class__.__name__} with volume of {self.volume} l."

    @property
    @abstractmethod
    def color(self) -> str:
        ...

potions/green.py:

from .base import Potion

class GreenPotion(Potion):
    color = "green"

potions/brown.py:

from .base import Potion

class BrownPotion(Potion):
    color = "brown"

potions/__init__.py:

from .base import Potion
from .brown import GreenPotion
from .brown import BrownPotion

usage.py:

from potions import GreenPotion, BrownPotion

b1 = GreenPotion(5)
b2 = BrownPotion(111)

b3 = b1 + b2
assert b3.volume == 116
assert type(b3) is BrownPotion

b4 = b1 * 3
assert b4.volume == 15
assert type(b4) is GreenPotion

b5 = b2 * 3
assert b5.volume == 333
assert type(b5) is BrownPotion

b6 = -b1
assert b6.volume == 5
assert type(b6) is BrownPotion

Running usage.py gives the following ImportError:

ImportError                               Traceback (most recent call last)
usage.py in <module>
----> 1 from potions import GreenPotion, BrownPotion
      2 
      3 b1 = GreenPotion(5)
      4 b2 = BrownPotion(111)
      5 

potions\__init__.py in <module>
----> 1 from .green import GreenPotion
      2 from .brown import BrownPotion

potions\brown.py in <module>
----> 1 from .base import Potion
      2 
      3 class GreenPotion(Potion):
      4     color = "green"

potions\base.py in <module>
      1 from abc import ABC, abstractmethod
      2 
----> 3 from .arithmatic import PotionArithmatic
      4  

potions\arithmatic.py in <module>
----> 1 from . import base, brown, green
      2 
      3 class PotionArithmatic:
      4     def __add__(self, other):

potions\green.py in <module>
----> 1 from .base import Potion
      2 
      3 class GreenPotion(Potion):
      4     color = "green"

ImportError: cannot import name 'Potion' from partially initialized module 'potions.base' (most likely due to a circular import) (potions\base.py)

Further analysis

Possible solutions

I've looked for hours and hours into this type of problem.

arithmatic.py:

def add_potion_instances(potion1, potion2):
    from . import base, brown, green # <-- added imports here
    return brown.BrownPotion(potion1.volume + potion2.volume)

class PotionArithmatic:
    def __add__(self, other):
        from . import base, brown, green # <-- added imports here
        # Adding potions always returns a brown potion.
        if isinstance(other, base.Potion):
            return add_potion_instances(self, other)
        return brown.BrownPotion(self.volume + other)

    def __mul__(self, other):
        from . import base, brown, green # <-- added imports here
        # Multiplying a potion with a number scales it.
        if isinstance(other, base.Potion):
            raise TypeError("Cannot multiply Potions")
        return self.__class__(self.volume * other)

    def __neg__(self):
        from . import base, brown, green # <-- added imports here
        # Negating a potion changes its color but not its volume.
        if isinstance(self, green.GreenPotion):
            return brown.BrownPotion(self.volume)
        else:  # isinstance(self, BrownPotion):
            return green.GreenPotion(self.volume)

Though this works, you can imagine this results in many additional lines if the file contains many more methods for the mixin class, esp. if these in turn call functions on the module's top level.

Many thanks!


Solution

  • TLDR: Rule of thumb

    you should not use on the mixin/inheritance architecture if the mixin returns an instance of the class (or of one of its descendants). In that case, the methods should be appended to the class object itself.

    Details: Solutions

    I thought of 2 (very similar) ways to get it to work. None is ideal, but they both seem to resolve the problem, by no longer relying on inheritance for the mixin.

    In both, the potions/base.py file is changed to the following:

    potions/base.py:

    from abc import ABC, abstractmethod
    
    class Potion(ABC): # <-- mixin is gone
        # (nothing changed here)
    
    from . import arithmatic  # <-- moved to the end
    arithmatic.append_methods()  # <-- explicitly 'do the thing'
    

    What we do with potions/arithmatic.py depends on the solution.

    Keep the mixin class, but append the methods manually

    This solution I like the best. In arithmatic.py, we can keep the original PotionArithmatic class. We just add a list of relevant dunder methods it, and the append_methods() function to do the appending.

    potions/arithmatic.py:

    from . import base, brown, green
    
    def add_potion_instances(potion1, potion2):
        # (nothing changed here)
    
    def PotionArithmatic:
        ATTRIBUTES = ["__add__", "__mul__", "__neg__"] # <-- this is new
        # (nothing else changed here)
    
    def append_methods(): # <-- this is new as well
        for attr in PotionArithmatic.ATTRIBUTES:
            setattr(base.Potion, attr, getattr(PotionArithmatic, attr))
    

    Completely get rid of the mixin

    Alternatively, we can get rid of the PotionArithmatic class alltogether, and just append the methods directly to the Potion class object:

    potions/arithmatic.py:

    from . import base, brown, green
    
    def _add_potion_instances(potion1, potion2):
        return brown.BrownPotion(potion1.volume + potion2.volume)
    
    def _ext_add(self, other):
        # Adding potions always returns a brown potion.
        if isinstance(other, base.Potion):
            return _add_potion_instances(self, other)
        return brown.BrownPotion(self.volume + other)
    
    def _ext_mul(self, other):
        # Multiplying a potion with a number scales it.
        if isinstance(other, base.Potion):
            raise TypeError("Cannot multiply Potions")
        return self.__class__(self.volume * other)
    
    def _ext_neg(self):
        # Negating a potion changes its color but not its volume.
        if isinstance(self, green.GreenPotion):
            return brown.BrownPotion(self.volume)
        else:  # isinstance(self, BrownPotion):
            return green.GreenPotion(self.volume)
    
    def append_methods():
        base.Potion.__add__ = _ext_add
        base.Potion.__mul__ = _ext_mul
        base.Potion.__neg__ = _ext_neg
    

    Consequences

    Both solutions work, but be aware that

    (a) they introduce more coupling and necessitate moving imports to the end of base.py, and

    (b) the IDE will no longer know about these methods when writing code, as they are added at run-time.