how to resolve latency issue with django M2M and filter_horizontal in ModelAdmin panel?

I have used django ModelAdmin with M2M relationship and formfield filtering code as follows: But for superuser or any other login where the number of mailboxes is more than 100k. I have sliced the available after filtering. But loading the m2m field takes time and times out for superuser login:

def formfield_for_manytomany(self, db_field, request, **kwargs):
    if db_field.name == "mailboxes":
        if request.user.is_superuser:
            queryset = Mailbox.objects.prefetch_related('domain').only('id','email')
            kwargs["queryset"] = queryset
            field = super().formfield_for_manytomany(db_field, request, **kwargs)
            field.widget.choices.queryset = queryset  # Limit visible options
            return field

        if request.user.groups.filter(name__in=['customers']).exists():
            queryset = Mailbox.objects.filter(
                domain__customer__email=request.user.email).prefetch_related('domain').only('id','email')
            kwargs["queryset"] = queryset
            field = super().formfield_for_manytomany(db_field, request, **kwargs)
            field.widget.choices.queryset = queryset
            return field
        return super().formfield_for_manytomany(db_field, request, **kwargs)

I want to use filter_horizontal only and not django auto_complete_light or any javascript. how can the latency be resolved. As you can see the queryset filtering is already done to get valid options. Slicing removed

the mailbox model is simple:

class Mailbox(AbstractPerson):
    username = models.EmailField(verbose_name='email', blank=True)
    email = models.EmailField(verbose_name='email', null=True,blank=True, unique=True)
        local_part = models.CharField(max_length=100,verbose_name='user part',help_text=hlocal_part)
    domain = models.ForeignKey(Domain, on_delete=models.CASCADE)

which has M2M relation with GroupMailIds model:

class GroupMailIds(models.Model):
    local_part = models.CharField(max_length=100,verbose_name='local part',help_text=hlocal_part)
    address = models.EmailField(unique=True,verbose_name='Email id of the distribution list')
    domain = models.ForeignKey(Domain, on_delete=models.CASCADE,related_name='domains')
    mailboxes = models.ManyToManyField(Mailbox,related_name='my_mailboxes')

Solution

Limited the number of objects loading during form init as:

class GroupMailIdsForm(forms.ModelForm):
    class Meta:
        model = GroupMailIds
        fields = "__all__"
                
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        request = getattr(self, 'request', None)
        if not request:
            return

        email = request.user.email
        qs = Mailbox.objects.all()

        if request.user.is_superuser:
            filtered_qs = qs
        else:
            filtered_qs = qs.none()  # No access if none of the conditions match

        # Set the filtered queryset to the form field
        self.fields['mailboxes'].queryset = filtered_qs

In ModelAdmin:

def get_form(self, request, obj=None, **kwargs):
    # Pass the request to the form
    form = super().get_form(request, obj, **kwargs)
    form.request = request  # Attach the request to the form
    return form

The init for non-superuser initializes the queryset with None and during the admin, populates the dropdown with required filtered values during admin based on login value