pythonpandasmatplotlib

How to fix the error when try to plot data with pandas?


I have encountered an issue in python source code for DETR model from Facebook(Archived repository). The problem is with pandas when it tries to plot something. I did not see this error before, but I think the pre-installed Google Colab packages cause this issue. The error is with these lines of codes(on else part) located in util/plot_utils.py:

for df, color in zip(dfs, sns.color_palette(n_colors=len(logs))):
    for j, field in enumerate(fields):
        if field == 'mAP':
            coco_eval = pd.DataFrame(
                np.stack(df.test_coco_eval_bbox.dropna().values)[:, 1]
            ).ewm(com=ewm_col).mean()
            axs[j].plot(coco_eval, c=color)
        else:
            
            df.interpolate().ewm(com=ewm_col).mean().plot(
                y=[f'train_{field}', f'test_{field}'],
                ax=axs[j],
                color='blue',
                style=['-', '--']
            )

This code tries to plot some figures based on these raw file:

{"train_lr": 0.00010000000000000003, "train_class_error": 60.685014724731445, "train_loss": 11.764228752681188, "train_loss_ce": 0.719119736126491, "train_loss_bbox": 0.4253895453044346, "train_loss_giou": 0.7821782167468753, "train_loss_ce_0": 0.7641034339155469, "train_loss_bbox_0": 0.4464654347726277, "train_loss_giou_0": 0.8194219555173602, "train_loss_ce_1": 0.7384649557726723, "train_loss_bbox_1": 0.43022043790136066, "train_loss_giou_1": 0.8033964250768934, "train_loss_ce_2": 0.740848937204906, "train_loss_bbox_2": 0.42099533123629435, "train_loss_giou_2": 0.7929297387599945, "train_loss_ce_3": 0.7324359204087939, "train_loss_bbox_3": 0.4274384592260633, "train_loss_giou_3": 0.7886527244533811, "train_loss_ce_4": 0.723468439919608, "train_loss_bbox_4": 0.42732386929648264, "train_loss_giou_4": 0.7813752229724612, "train_loss_ce_unscaled": 0.719119736126491, "train_class_error_unscaled": 60.685014724731445, "train_loss_bbox_unscaled": 0.08507791001881872, "train_loss_giou_unscaled": 0.39108910837343763, "train_cardinality_error_unscaled": 22.946428571428573, "train_loss_ce_0_unscaled": 0.7641034339155469, "train_loss_bbox_0_unscaled": 0.08929308610303062, "train_loss_giou_0_unscaled": 0.4097109777586801, "train_cardinality_error_0_unscaled": 14.321428571428571, "train_loss_ce_1_unscaled": 0.7384649557726723, "train_loss_bbox_1_unscaled": 0.0860440872077431, "train_loss_giou_1_unscaled": 0.4016982125384467, "train_cardinality_error_1_unscaled": 16.142857142857142, "train_loss_ce_2_unscaled": 0.740848937204906, "train_loss_bbox_2_unscaled": 0.08419906494340726, "train_loss_giou_2_unscaled": 0.39646486937999725, "train_cardinality_error_2_unscaled": 20.535714285714285, "train_loss_ce_3_unscaled": 0.7324359204087939, "train_loss_bbox_3_unscaled": 0.0854876914194652, "train_loss_giou_3_unscaled": 0.39432636222669054, "train_cardinality_error_3_unscaled": 19.821428571428573, "train_loss_ce_4_unscaled": 0.723468439919608, "train_loss_bbox_4_unscaled": 0.08546477396573339, "train_loss_giou_4_unscaled": 0.3906876114862306, "train_cardinality_error_4_unscaled": 21.642857142857142, "test_class_error": 25.03192901611328, "test_loss": 7.842764377593994, "test_loss_ce": 0.5921314656734467, "test_loss_bbox": 0.17633574455976486, "test_loss_giou": 0.4800571948289871, "test_loss_ce_0": 0.7295672297477722, "test_loss_bbox_0": 0.1843198761343956, "test_loss_giou_0": 0.5027516782283783, "test_loss_ce_1": 0.650816947221756, "test_loss_bbox_1": 0.17886291444301605, "test_loss_giou_1": 0.5067266523838043, "test_loss_ce_2": 0.6202048659324646, "test_loss_bbox_2": 0.179609976708889, "test_loss_giou_2": 0.5068651139736176, "test_loss_ce_3": 0.6036751568317413, "test_loss_bbox_3": 0.1801770180463791, "test_loss_giou_3": 0.49125969409942627, "test_loss_ce_4": 0.595748633146286, "test_loss_bbox_4": 0.177798293530941, "test_loss_giou_4": 0.4858558773994446, "test_loss_ce_unscaled": 0.5921314656734467, "test_class_error_unscaled": 25.03192901611328, "test_loss_bbox_unscaled": 0.03526714816689491, "test_loss_giou_unscaled": 0.24002859741449356, "test_cardinality_error_unscaled": 21.375, "test_loss_ce_0_unscaled": 0.7295672297477722, "test_loss_bbox_0_unscaled": 0.03686397522687912, "test_loss_giou_0_unscaled": 0.25137583911418915, "test_cardinality_error_0_unscaled": 8.125, "test_loss_ce_1_unscaled": 0.650816947221756, "test_loss_bbox_1_unscaled": 0.03577258251607418, "test_loss_giou_1_unscaled": 0.25336332619190216, "test_cardinality_error_1_unscaled": 20.25, "test_loss_ce_2_unscaled": 0.6202048659324646, "test_loss_bbox_2_unscaled": 0.03592199645936489, "test_loss_giou_2_unscaled": 0.2534325569868088, "test_cardinality_error_2_unscaled": 25.5, "test_loss_ce_3_unscaled": 0.6036751568317413, "test_loss_bbox_3_unscaled": 0.03603540360927582, "test_loss_giou_3_unscaled": 0.24562984704971313, "test_cardinality_error_3_unscaled": 22.125, "test_loss_ce_4_unscaled": 0.595748633146286, "test_loss_bbox_4_unscaled": 0.03555965796113014, "test_loss_giou_4_unscaled": 0.2429279386997223, "test_cardinality_error_4_unscaled": 22.125, "test_coco_eval_bbox": [0.25752657647843996, 0.4695548915331907, 0.2376618407445379, -1.0, 0.23612653129171618, 0.3743419813637686, 0.04096385538578033, 0.2349397599697113, 0.5843373503535986, -1.0, 0.5877192996442318, 0.5769230775535107], "epoch": 0, "n_parameters": 60219142}

Error message:

/content/detr/util/plot_utils.py:66: FutureWarning: DataFrame.interpolate with object dtype is deprecated and will raise in a future version. Call obj.infer_objects(copy=False) before interpolating instead.
  df.interpolate().ewm(com=ewm_col).mean().plot(
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
TypeError: float() argument must be a string or a real number, not 'list'

The above exception was the direct cause of the following exception:

ValueError                                Traceback (most recent call last)
/usr/local/lib/python3.10/dist-packages/pandas/core/window/rolling.py in _prep_values(self, values)
    369             else:
--> 370                 values = ensure_float64(values)
    371         except (ValueError, TypeError) as err:

pandas/_libs/algos_common_helper.pxi in pandas._libs.algos.ensure_float64()

ValueError: setting an array element with a sequence.

The above exception was the direct cause of the following exception:

TypeError                                 Traceback (most recent call last)
6 frames
/usr/local/lib/python3.10/dist-packages/pandas/core/window/rolling.py in _apply_blockwise(self, homogeneous_func, name, numeric_only)
    486             try:
--> 487                 arr = self._prep_values(arr)
    488             except (TypeError, NotImplementedError) as err:

/usr/local/lib/python3.10/dist-packages/pandas/core/window/rolling.py in _prep_values(self, values)
    371         except (ValueError, TypeError) as err:
--> 372             raise TypeError(f"cannot handle this type -> {values.dtype}") from err
    373 

TypeError: cannot handle this type -> object

The above exception was the direct cause of the following exception:

DataError                                 Traceback (most recent call last)
<ipython-input-9-56bcedec1dc6> in <cell line: 6>()
      4 from pathlib import Path, PurePath
      5 
----> 6 plot_logs([
      7     Path("/content/drive/MyDrive/Colab Notebooks/Datasets/dataset/model")
      8     ])

/content/detr/util/plot_utils.py in plot_logs(logs, fields, ewm_col, log_name)
     64             else:
     65                 df = df.infer_objects()
---> 66                 df.interpolate().ewm(com=ewm_col).mean().plot(
     67                     y=[f'train_{field}', f'test_{field}'],
     68                     ax=axs[j],

/usr/local/lib/python3.10/dist-packages/pandas/core/window/ewm.py in mean(self, numeric_only, engine, engine_kwargs)
    553                 normalize=True,
    554             )
--> 555             return self._apply(window_func, name="mean", numeric_only=numeric_only)
    556         else:
    557             raise ValueError("engine must be either 'numba' or 'cython'")

/usr/local/lib/python3.10/dist-packages/pandas/core/window/rolling.py in _apply(self, func, name, numeric_only, numba_args, **kwargs)
    615 
    616         if self.method == "single":
--> 617             return self._apply_blockwise(homogeneous_func, name, numeric_only)
    618         else:
    619             return self._apply_tablewise(homogeneous_func, name, numeric_only)

/usr/local/lib/python3.10/dist-packages/pandas/core/window/rolling.py in _apply_blockwise(self, homogeneous_func, name, numeric_only)
    487                 arr = self._prep_values(arr)
    488             except (TypeError, NotImplementedError) as err:
--> 489                 raise DataError(
    490                     f"Cannot aggregate non-numeric type: {arr.dtype}"
    491                 ) from err

DataError: Cannot aggregate non-numeric type: object

How can I fix this problem?


Solution

  • try

    df[
        [f'train_{field}', f'test_{field}']
    ].interpolate().ewm(com=ewm_col).mean().plot(
        y=[f'train_{field}', f'test_{field}'],
        ax=axs[j],
        color='blue',
        style=['-', '--']
    )
    

    Problem is not with plot.

    There are columns with type object in your DataFrame (test_coco_eval_bbox probably) and pandas cannot interpolate them