pythonjupyter-notebookgoogle-kubernetes-enginejupyterhub

Unable Spawn new notebook using Custom Kubespawner in Jupyterhub


I am facing an issue in spawning new pod with jupyterhub using custom kubespawner. Below is my custom_kubespawner.py file

import json
import os
from kubespawner import KubeSpawner

class CustomSpawner(KubeSpawner):
def __init__(self, *args, **kwargs):
    super().__init__(*args, **kwargs)
    
def load_config(self):
    config_file_path = "/srv/jupyterhub/profiles.json"  # Update this to your file path
    if os.path.exists(config_file_path):
        with open(config_file_path) as f:
            self.lab_config = json.load(f)
    else:
        raise FileNotFoundError(f"Configuration file not found: {config_file_path}")

async def start(self):
    # Load the configuration before starting the pod
    self.load_config()

    # Use the loaded configuration to set the parameters
    self.singleuser_image = self.lab_config.get("notebook_image", "jupyter/notebook")
    self.mem_limit = self.lab_config.get("mem_limit", "2G")
    self.cpu_limit = self.lab_config.get("cpu_limit", 1)

    # Set environment variables
    env_vars = self.lab_config.get("env_variables", {})
    for key, value in env_vars.items():
        self.environment[key] = value
    
    # Set affinity based on the JSON configuration
    affinity = self.lab_config.get("affinity", {})
    if affinity:
        self.extra_pod_config['affinity'] = affinity
    
    # Set the storage capacity and mount path
    self.storage_capacity = self.lab_config.get("storage_capacity", "20Gi")  # Default to 20Gi if not specified
    self.pvc_name_template = f"{self.user.name}-pvc"
    self.volume_name_template = f"{self.user.name}-vol"
    self.volumes = [

    {

        "name": self.volume_name_template,

        "persistentVolumeClaim": {"claimName": "{pvc_name}"},

    }

    ]

    self.volume_mounts = [
        {
            'mountPath': '/home/jovyan',  # Specify where to mount the volume in the pod
            'name': self.volume_name_template
        }
    ]

    return await super().start()

The Jupyterhub_cnnfig.py file is having the code which is available jupyter_config.py

I have added the lines

sys.path.insert(0, "/srv/jupyterhub")
sys.path.insert(0, "/usr/local/etc/jupyterhub")
from custom_spawner import CustomSpawner
c.JupyterHub.spawner_class = "CustomSpawner"

to jupyterhub_config.py file import and use the custom Spawner.

{
"notebook_image": "jupyter/scipy-notebook",
"cpu_limit": 1,
"mem_limit": "2G",
"env_variables": {
    "MY_ENV_VAR": "TOKYO"
},
"affinity": {
    "nodeAffinity": {
        "requiredDuringSchedulingIgnoredDuringExecution": {
            "nodeSelectorTerms": [
                {
                    "matchExpressions": [
                        {
                            "key": "image",
                            "operator": "In",
                            "values": [
                                "minimal-notebook"
                            ]
                        }
                    ]
                }
            ]
        }
    }
},
"storage_capacity": "10Gi"
}

All the there files are stored in /srv/jupyterhub folde rin the hub pod

[E 2024-10-28 16:09:54.115 JupyterHub gen:629] Exception in Future <Task finished name='Task-131' coro=<BaseHandler.spawn_single_user.<locals>.finish_user_spawn() done, defined at /usr/local/lib/python3.11/site-packages/jupyterhub/handlers/base.py:1081> exception=KeyError('pvc_name')> after timeout
    Traceback (most recent call last):
      File "/usr/local/lib/python3.11/site-packages/tornado/gen.py", line 624, in error_callback
        future.result()
      File "/usr/local/lib/python3.11/site-packages/jupyterhub/handlers/base.py", line 1088, in finish_user_spawn
        await spawn_future
      File "/usr/local/lib/python3.11/site-packages/jupyterhub/user.py", line 905, in spawn
        raise e
      File "/usr/local/lib/python3.11/site-packages/jupyterhub/user.py", line 801, in spawn
        url = await gen.with_timeout(timedelta(seconds=spawner.start_timeout), f)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/srv/jupyterhub/custom_spawner.py", line 117, in start
        return await super().start()
               ^^^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 2718, in _start
        pod = await self.get_pod_manifest()
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 2015, in get_pod_manifest
        volumes=self._expand_all(self.volumes),
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1845, in _expand_all
        return [self._expand_all(i) for i in src]
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1845, in <listcomp>
        return [self._expand_all(i) for i in src]
                ^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1847, in _expand_all
        return {k: self._expand_all(v) for k, v in src.items()}
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1847, in <dictcomp>
        return {k: self._expand_all(v) for k, v in src.items()}
                   ^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1847, in _expand_all
        return {k: self._expand_all(v) for k, v in src.items()}
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1847, in <dictcomp>
        return {k: self._expand_all(v) for k, v in src.items()}
                   ^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1849, in _expand_all
        return self._expand_user_properties(src)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
      File "/usr/local/lib/python3.11/site-packages/kubespawner/spawner.py", line 1827, in _expand_user_properties
        rendered = template.format(
                   ^^^^^^^^^^^^^^^^
    KeyError: 'pvc_name'

This above is the error.

I have deployed jupthub of GKE using the to 0 to jupytehub concept.

Any update or alterntive way to achive this, Note: don't want to use profileList since I need to provision based on input provided, so I cannot show the options to the user.


Solution

  • It looks like the KeyError: 'pvc_name' says that the template string {pvc_name} in the volumes configuration isn’t being populated correctly. This may be due to pvc_name isn’t defined in the current scope of the spawner.

    You can directly refer to the self.pvc_name_template variable in your volumes configuration, replacing "{pvc_name}" with self.pvc_name_template. Here’s an updated version of the relevant code:

    self.volumes = [
        {
            "name": self.volume_name_template,
            "persistentVolumeClaim": {"claimName": self.pvc_name_template},
        }
    ]
    

    Also, ensure that self.pvc_name_template and self.volume_name_template are correctly assigned in the spawner instance and that these templates match any references within your storage configuration.