What is max_evals in hyperopt fmin function?

I'm trying to run a multi-classification problem. I have run a baseline lightGBM model with around 80% accuracy rate. I'm trying to further fine-tuning its hyperparameter using Hyperopt. However, when configuring Fmin, I'm confused in how should I set the max_evals to be, and what exactly is max_evals?

from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from hyperopt.pyll.base import scope
# Load randomized search best parameters
with open('best_params.json', 'r') as file:
    best_params = json.load(file)

print("Best hyperparameters loaded from best_params.json")
# Define the search space around the best parameters
search_space = {
    'num_leaves': scope.int(hp.quniform('num_leaves', best_params['num_leaves'] - 10, best_params['num_leaves'] + 10, 1)),
    'learning_rate': hp.uniform('learning_rate', best_params['learning_rate'] * 0.8, best_params['learning_rate'] * 1.2),
    'n_estimators': scope.int(hp.quniform('n_estimators', best_params['n_estimators'] - 100, best_params['n_estimators'] + 100, 10)),
    'min_child_weight': hp.uniform('min_child_weight', best_params['min_child_weight'] * 0.8, best_params['min_child_weight'] * 1.2),
    'subsample': hp.uniform('subsample', max(0.5, best_params['subsample'] * 0.8), min(1.0, best_params['subsample'] * 1.2)),
    'colsample_bytree': hp.uniform('colsample_bytree', max(0.5, best_params['colsample_bytree'] * 0.8), min(1.0, best_params['colsample_bytree'] * 1.2)),
    'reg_alpha': hp.uniform('reg_alpha', best_params['reg_alpha'] * 0.8, best_params['reg_alpha'] * 1.2),
    'reg_lambda': hp.uniform('reg_lambda', best_params['reg_lambda'] * 0.8, best_params['reg_lambda'] * 1.2)
}

# Define the objective function
def objective(params):
    model = lgb.LGBMClassifier(objective='multiclass', num_class=len(set(y_train)), **params)
    score = cross_val_score(model, X_train, y_train, cv=3, scoring='accuracy').mean()
    return {'loss': -score, 'status': STATUS_OK}

# Run the optimization
trials = Trials()
best_hyperparams = fmin(fn=objective,
                        space=search_space,
                        algo=tpe.suggest,
                        max_evals=50,  # Adjust the number of evaluations based on your needs
                        trials=trials,
                        rstate=np.random.default_rng(42))

print("Best hyperparameters found using Hyperopt: ", best_hyperparams)

hyperopt attempts multiple "trials". In each trial, it uses one possible combination of hyperparameters and measures the output of some objective function given those hyperparameters as input.

The "space" you provide defines the set of all possible hyperparameter combinations to try.

max_evals indicates the maximum number of such trials to conduct. It's written as "maximum" because hyperopt may sometimes conduct fewer trials, for example if early stopping is triggered in the search process.

Here's a minimal, reproducible example using hyperopt==0.2.7, lightgbm==4.5.0, and scikit-learn==1.6.0.

from hyperopt import fmin, tpe, hp, STATUS_OK, Trials
import lightgbm as lgb
import numpy as np
from hyperopt.pyll.base import scope
from sklearn.model_selection import cross_val_score
from sklearn.datasets import make_blobs


class HyperoptObjective:
    def __init__(self, X, y):
        self.X = X
        self.y = y

    def __call__(self, params):
        model = lgb.LGBMClassifier(
            objective="multiclass",
            num_class=len(set(self.y)),
            **params
        )
        score = cross_val_score(
            estimator=model, X=self.X, y=self.y, cv=3, 
            scoring="accuracy"
        ).mean()
        return {"loss": -score, "status": STATUS_OK}

# create a 3-class classification dataset
X, y = make_blobs(n_samples=10_000, n_features=5, centers=3)

# space with 4 possible combinations:
#
#   num_leaves=40, n_estimators=10
#   num_leaves=40, n_estimators=20
#   num_leaves=50, n_estimators=10
#   num_leaves=50, n_estimators=20
#
space = {
    "num_leaves": scope.int(hp.quniform("num_leaves", 40, 50, 10)),
    "n_estimators": scope.int(hp.quniform("n_estimators", 10, 20, 10)),
}

# create an object to hold trial results
trials = Trials()

# run a sweep
num_trials = 2
best_hyperparams = fmin(
    fn=HyperoptObjective(X, y),
    space=space,
    algo=tpe.suggest,
    max_evals=num_trials,
    trials=trials,
    rstate=np.random.default_rng(708),
)

print(f"Best hyperparameters found ({num_trials} trials):\n{best_hyperparams}")
# Best hyperparameters found (2 trials):
# {'n_estimators': 10.0, 'num_leaves': 40.0}

print(f"trials run:\n{trials.vals}")
# trials run:
# {'n_estimators': [10.0, 20.0], 'num_leaves': [40.0, 40.0]}

Notice that there are 4 possible combinations of the input parameters in space, but setting max_evals=2 resulted in only 2 of them being tried.

how should I set the max_evals

The answer to this depends a lot on your specific application.

For example, if conducting a single trial is very expensive (in terms of time, memory, compute, or money), you may want a relatively low value of max_evals.

For more advice on how to set that value to achieve good performance, it would probably be better to ask on https://stats.stackexchange.com/.