Use setup.py / pyproject.toml to compile a library also in editable install

I am setting up a Python package with setuptools, together with pyproject.toml. The Python code is dependent on a C library that needs to be compiled and installed alongside the code (it's a make project).

I have put something together that works for a pip install . and also for python -m build to make a distributable:

# pyproject.toml

[project]
name = "mypackage"

[build-system]
requires = ["setuptools >= 61.0", "wheel"]
build-backend = "setuptools.build_meta"

[tool.setuptools]
packages = ["mypackage"]
package-dir = { "" = "src" }

# setup.py

from pathlib import Path
from setuptools import setup
from setuptools.command.install import install
from setuptools.command.develop import develop
from setuptools.command.build import build
import os
import subprocess


mylib_relative = "mylib"
mylib_root = Path(__file__).parent.absolute() / mylib_relative


def create_binaries():
    subprocess.call(["make", "-C", mylib_relative])


def remove_binaries():
    patterns = (
        "*.a",
        "**/*.o",
        "*.bin",
        "*.so",
    )
    for pattern in patterns:
        for file in mylib_root.glob(pattern):
            os.remove(file)


class CustomBuild(build):
    def run(self):
        print("\nCustomBuild!\n")

        remove_binaries()
        create_binaries()
        super().run()


class CustomDevelop(develop):
    def run(self):
        print("\nCustomDevelop!\n")

        remove_binaries()
        create_binaries()
        super().run()


class CustomInstall(install):
    def run(self):

        print("\n\nCustomInstall\n\n")

        mylib_lib = mylib_root / "adslib.so"
        mylib_dest = Path(self.install_lib)
        if not mylib_dest.exists():
            mylib_dest.mkdir(parents=True)
        self.copy_file(
            str(mylib_lib),
            str(mylib_dest),
        )
        super().run()


setup(
    cmdclass={
        "build": CustomBuild,
        "develop": CustomDevelop,
        "install": CustomInstall,
    },
)

However, when I make an editable install with pip with pip install -e . [-v], the library is not compiled and installed, only the Python source is added to the venv path. But the package won't work without the library.

You can see I already added the develop command in setup.py, but it looks like it's never called at all.

How can I customize the editable install to also compile my library first?

I found a solution / workaround I can work with. After pip install -e . the directory containing your package (typically src/) will be appended to PATH (tested by printing sys.path from my package).

So if I just make sure my compiled library is put in src/ it will also be available on PATH after an editable install, just as if it is installed normally. If the .so file is under .gitignore it shouldn't bother anyone by being in src/ instead of the library directory.

Full setup.py:

from pathlib import Path
from setuptools import setup
from setuptools.command.install import install
from setuptools.command.build_py import build_py
import os
import subprocess


src_folder = Path(__file__).parent.absolute() / "src"
# ^ This will be on PATH for editable install
mylib_folder = Path(__file__).parent.absolute() / "mylib"
mylib_file = src_folder / "mylib.so"


class CustomBuildPy(build_py):
    """Custom command for `build_py`.

    This command class is used because it is always run, also for an editable install.
    """

    @classmethod
    def compile_mylib(cls):
        """Return `True` if mylib was actually compiled."""
        cls._clean_library()
        cls._compile_library()

    @staticmethod
    def _compile_library():
        """Use `make` to build mylib - build is done in-place."""
        # Produce `mylib.so`:
        subprocess.call(["make", "-C", "mylib"])

    @staticmethod
    def _clean_library():
        """Remove all compilation artifacts."""
        patterns = (
            "*.a",
            "**/*.o",
            "*.bin",
            "*.so",
        )
        for pattern in patterns:
            for file in mylib_folder.glob(pattern):
                os.remove(file)

        if mylib_file.is_file():
            os.remove(mylib_file)

    def run(self):
        # Move .so file into src/ to have it on PATH:
        self.compile_adslib()
        self.move_file(
            str(mylib_folder / "mylib.so"),
            str(mylib_file),
        )

        super().run()


class CustomInstall(install):
    """Install compiled mylib (but only for Linux)."""
    def run(self):
        mylib_dest = Path(self.install_lib)
        if not mylib_dest.exists():
            mylib_dest.mkdir(parents=True)
        self.copy_file(
            str(mylib_file),
            str(mylib_dest),
        )
        super().run()


setup(
    cmdclass={
        "build_py": CustomBuildPy,
        "install": CustomInstall,
    },
)
# See `pyproject.toml` for all package information

# Also see `MANIFEST.in`

This is actually for the pyads package and the full PR can be seen here: https://github.com/stlehmann/pyads/pull/426