pythoncsegmentation-faultgarbage-collectionpython-extensions

segmentation fault Segfault python extension in c++

This extension is used to search data in a linked list and return it as a nested tuple of lists.[(ele1,ele2,),...]

However when the program runs for a period of time, segment fault will occur. I know that there might be something I'm missing with python memory management, but it's not clear to me exactly where the problem is.

header file:

#include "Python.h"
#define  PY_EXTENSION_H
#ifndef TESTFORSCORE_MAIN_H
#define TESTFORSCORE_MAIN_H
#define MAX(a,b) a<b?b:a
#define MIN(a,b) a<b?a:b
typedef unsigned char byte;
typedef struct{
    char* content;
    int content_len;
    int features_len;
    byte* features;
    long id;
}NODE;

typedef struct LISTNODE{
    LISTNODE* next;
    NODE* node;
}LISTNODE;

extern "C"{
static PyObject *compare(byte* features1,byte* features2,int min);
static PyObject *init_map(byte* features,int features_len,char* content ,int content_len,long id);
static PyObject *get_list_size(PyObject *self, PyObject *args);
static PyObject *search(PyObject *self, PyObject *args);
}
#endif

cpp

#define PY_SSIZE_T_CLEAN
#include "example.h"


static LISTNODE *head_of_map=(LISTNODE*) PyMem_RawMalloc(sizeof(LISTNODE));
LISTNODE *current_head=head_of_map;
long COUNTER=0;
static PyObject *get_list_size(PyObject *self, PyObject *args)
{
    return Py_BuildValue("l", COUNTER);
}
static PyObject *init_map(PyObject *self, PyObject *args){//
    PyObject  *obj1,*obj2;
    int len1,len2;
    long activity;
    byte *features,*tmp_b;
    char *content,*tmp_c;
    if (!PyArg_ParseTuple(args,"SSiiL", &obj1,&obj2,&len1,&len2,&activity)) {
        return NULL;
    }

    features=(byte *) PyBytes_AsString(obj1);
    content=PyBytes_AsString(obj2);

    LISTNODE *list_node=(LISTNODE*) PyMem_RawMalloc(sizeof(LISTNODE));
    NODE *node=(NODE*) PyMem_RawMalloc(sizeof(NODE));
    tmp_b=(byte *) PyMem_RawMalloc(sizeof(byte)*len1);
    tmp_c=(char *) PyMem_RawMalloc(sizeof(char)*len2);
    memcpy(tmp_b,features,len1);
    memcpy(tmp_c,content,len2);
    node->features=tmp_b;
    node->content=tmp_c;
    node->features_len=len1;
    node->content_len=len2;
    node->id=activity;
    list_node->node=node;
    current_head->next=list_node;
    current_head=current_head->next;
    COUNTER+=1;
    // err maybe here
    //Py_DecRef(obj1);
    //Py_DecRef(obj2);
    PyMem_FREE(tmp_b);
    PyMem_Free(tmp_c);
    return Py_BuildValue("b",0);
}

static double _compare(byte* features1,byte* features2,int min){
    //do something ...
    return   0.3;
}
static PyObject *search(PyObject *self, PyObject *args){//(byte* features,char *content,int features_len,int content_len,double threshold,int _idx){

    PyObject *obj1,*obj2,*tuple,*content,*matched_content,*score1,*_id,*matched_id, *lis= PyList_New(0);
    byte* features;
    char *c_content,*tmp_content,*tmp_matched;
    int content_len,length ,idx;
    double  threshold,score;
    LISTNODE *p = head_of_map;

    if (!PyArg_ParseTuple(args,"SSiidi", &obj1,&obj2,&length,&content_len,&threshold,&idx)) {

        return NULL;
    }
    features = (unsigned  char*)PyBytes_AsString(obj1);
    c_content = PyBytes_AsString(obj2);
//    Py_DecRef(obj1);
//    Py_DecRef(obj2);
    p=p->next;
    int counter=0;
    while (p!=NULL){
        counter+=1;
//      compare here
        score =0.3;
        if(score>threshold){
            //if(counter>1000) break;
            tuple      = PyTuple_New(5);
            score1     = Py_BuildValue("d",score);
            _id        = Py_BuildValue("b",idx);
            matched_id = Py_BuildValue("l",p->node->id);

            tmp_content = (char*)PyMem_Malloc(sizeof(char)*content_len);
            if (tmp_content == NULL)
                return PyErr_NoMemory();
            memcpy(tmp_content,c_content,content_len);
            content = Py_BuildValue("y#",tmp_content,(Py_ssize_t) sizeof(char)*content_len);

            tmp_matched=(char*) PyMem_Malloc(sizeof(char)*p->node->content_len);
            if (tmp_matched == NULL)
                return PyErr_NoMemory();
            memcpy(tmp_matched,p->node->content, p->node->content_len);
            matched_content = Py_BuildValue("y#",tmp_matched,(Py_ssize_t) sizeof(char)*p->node->content_len);

            PyTuple_SetItem(tuple,0,_id);
            PyTuple_SetItem(tuple,1,matched_id);
            PyTuple_SetItem(tuple,2,score1);
            PyTuple_SetItem(tuple,3,content);
            PyTuple_SetItem(tuple,4,matched_content);

            PyList_Append(lis,tuple);
            Py_DecRef(tuple);

            PyMem_Free(tmp_content);
            PyMem_Free(tmp_matched);
        }
        p=p->next;
    }

    if (PyErr_Occurred()){
        Py_DECREF(lis);
        return NULL ;
    }
    return lis;
}
static PyMethodDef exampleMethods[] = {
        {"get_list_size", get_list_size, METH_VARARGS, "example"},
        {"compare",compare,METH_VARARGS,"example"},
        {"init_map",init_map,METH_VARARGS,"example"},
        {"destory",destory,METH_VARARGS,""},
        {"search",search,METH_VARARGS,"example"},
        { NULL, NULL, 0, NULL}
};

static struct PyModuleDef ptexamplemodule = {
        PyModuleDef_HEAD_INIT,
        "example",           /* name of module */
        "A module that imports an API",  /* Doc string (may be NULL) */
        -1,                 /* Size of per-interpreter state or -1 */
        exampleMethods       /* Method table */
};
/* Module initialization function */
PyMODINIT_FUNC
PyInit_example(void) {
    PyObject *m;

    m = PyModule_Create(&ptexamplemodule);
    if (m == NULL)
        return NULL;

    /* Import sample, loading its API functions */
    return m;
}
int main() {
return 0;
}

setup file

from distutils.core import setup, Extension

setup(name='example',
      ext_modules=[
          Extension('example',
                    ['example.cpp'],
                    include_dirs = [],  # May need pysample.h directory
                    )
      ],
      version="0.0.1"
      )

test file

import example
import random
import time
import faulthandler
 
faulthandler.enable()
lis=[(b'\xd3\nR\x952\xbf\x8e\xeb[\xdc\xe8\xfb\xcb\x9f\xb4\xd2', b'\x856\x96\xda\xe4H_\xf9\xb3\x95\xff\xda\xc9\x05\x17\xd5'), (b'!\xdf\xc1\xc20h\xc3#\x8f\x8a\xd3\x92\xc9}\x1b\x08', b'\xa3\x03m\xd3\x96\x01\xb2\xe0\xfcOg\x87\xfa\xadA\x89'), (b'Fu\xe6\x97=\xb9\x88S\xe4Q\xc06\x9f[\x84J', b'`9\xf2- \x89\xc63?\xe3=\xf9o\x9b\xe4\x8e'), (b'\x95b\x1f\xba\x90\xfa\xe9j#\xd4\x12x\xc3\x93#\x1c', b'8f\xc4\xeb\xef\x95\xe8\x81\xec\xed1\x00j\xc2\xd2j'), (b'\xf9\x8bUk"\xa2y\xfbHI\x08\xf6\x03\xd3Ye', b'\xdf\xdb\xb6\x87\xf3\x05-p\x80%8\x8cd[43'), (b'\xee\x92\xcf\xb5\xd7\x05j\x92\xa2\x1c\xf7a.[\x05\x86', b'Y\x00|\x02\xa1\xb1wI\x08\xb3\xfc\xfc\xc7\xf5db'), (b"\x9d\xe0&\\~\xbf\xaf`\xae\xc2?'B\xfa\x95\x86", b'FL\xe3\xbb\x18\xeef\x08%\xe1\xc8,\xe3J=\xd5'), (b'\xe1\xfb\xc9n\x89\x1d\x9a\x9d\xe9\x1a\x9c\xde\xcd\xce\xbd\xcf', b'\xa7\xe0\x9b\x8bl\x88\x85,b\x04`3\xe6\x03\x85\x8e'), (b'\xc5\xbf\xd4\xa7\xe7e\xe5\x8a\xe0\xbc\xa8\xb8Yw\x0c\xd3', b":\xc2a\xb6\xd3\x1ct'z\xfe\xf9\xa7P\x8c\x1c\x7f"), (b'\xfd\xb2\x9aP\xc9\x0c\xcf\xe2\x8c\x82g\x8c{r\x94\x84', b'3B\xca\x8c\xd8\x9a\xb8\x94iyT\xd6\x05N_\x8a'), (b'\xea\x7f\x83\xaa\\\x10y\xd1h\x93l\xd9\xfe\x87\xe8j', b'jj\x8f\x1a-tq\xd9\xf1\xf8\xf7\xa5\xf6\x86\xb9\xdb'), (b'\xa4\x0b\x0f\xf1\x84\x95\x1eK\xddtH\xf0IaY:', b'p\xb5\xd9"\xb7\x0f(\r\xb1\x9bm\xc0hq\x00e'), (b'ga\x8f\xbbo\xff\xedQ\x9aM\xbe\x9cTO\xd35', b'=\xc3\x9d\x8d\xd0\xae8*1\x1d\xbf\x84\x89~76'), (b'\xaf\xe1%\xc6\xedo\xec24\xc0\x9d\xa8\x9fy=N', b'\x03m\x81\xd7B\x04\x1a*`C\xee\xccb\x1fm\xf0'), (b'\x1f\x10\xf6\x9fa\xc5\xbf\xaf\x18\xb1Mw\xba\x92&\xe1', b'\xb7\x90/\r$\t\xce}xk\xe7\xbeL\xe9\x8e\xc6'), (b'\xc8.\xd9)\x08g/\xc6\x0e\xed\xbb\xb9\xab\xad\xc8\xa1', b'\xa1c\xcc\xe9\xaasJ\x98\x1d\xd24\xc0\xf2+\x9d\xac'), (b'\xa1\x91\xa0:D\x11(\xb3\xbc0\x13\xd9\xdeD\xb9\x86', b'\x0fV\xa1\x8fl\x0c\xf4\x94=\xe7\xd0*\xdb\xcb\xa5;'), (b'bx;\x8dA\x83\xb8lca\x9c\xadV\xc2\xba/', b'\xe9\xe0\xe7({\x11\r*\xca\xf1\x08\x9bG\x830\x82'), (b"\xde\xbb\xd9\xce{%MP'Xj\xed8>\xa3a", b'\\5M\x13\x13\xb1YY\xa53y\x99$a\x91\xa5'), (b'\xf1\xda\xc4\xfe\xc6\x83\xe6\xf3\xc6\x88\x86H\xc4\x15\x12\xb4', b"\xf8'^\xe5\xb8`\x0fl\x06$\x1cz:\x93\xa1\\"), (b'\xc7@\x03Ny\x11\x96\xcb\xfe\x19\x0e\x18\xbf\xd59I', b'P\xf0\x0f\xec\xaf\x1bS\xa8\x8dt\x11O\xe8\x0f|2'), (b'\xb4\xda\xff\x8c0\x18/\xf1\xad\xa4F\x94\xb1\xad\xf4\xf1', b'.\x83(\x8f#\xe1\x11\x96\xdeS\x17\x8b\xe3]\xb5\xca'), (b'\x1d\t\x11\xc1\x15\xdc\x8e\xea\xe1\xd2\xba%J\\\xf2\xfd', b'\x070ht\xb4\xdeQ+\xc5\xb8#\xac\xd9z\xc0A'), (b' V\xe3\xc5gEg\xc5\xd5\xe1\xd6\xd7\xab\xd0\xb6n', b'\x88\x9b8\xf6\xe26\x92z\x91\xe4\x92\x1e\xbc\xc5;u'), (b'\x88}\xd7*QFn:\xd7\x12RA\xe6-\xd2)', b'\xeb\x04<\xe6\xfa)u\x9ds\xecj\xa7\x84\xcf\xa7\xa7'), (b"\xec\x04.bP)'\xb6\x08\x05\x92\x8c\x85\x1a\x84\x8f", b'@\x80\x08A\xba\x9c2Sd\xdf\x97\x07\xc4Im\xef'), (b'\x11(\x9ez\x0c\xd7\xca+7\xcc\xb0\x19,:\xcb5', b'\xf9V\x8cm\xe5-\xf9\xf1\xf6g\x86~\x8f\xd0<\x84'), (b'\xbb\xe9I=\xc74\xc9C\x0cT\x81\x9d\x85\xdeWF', b'\xfe\x1b\xb1gi\x94O\x98\xf0\x90cJ\xdb\xbf\xc7\x1e'), (b'2`\x9d\x9c\x02\xe4\xce\xf0\x14\xc0Z\xaaW\xf2\xa5\xff', b'\xe6\x05?\xe3X"\xb2O)~\x96\x8f^\x9c\x077'), (b'\xaf\x87,a\xa9\xcd\n|O\x1e\xa7\xdd\xba\x89;\x9d', b'\x98b\xac?\x13*\xf2m\xfeo=v\x96\xd5\x14\x8d'), (b'\xee\xcb{\xeeRQ\x82}\x05|\xc6S\x82\x85\xa4\xff', b'C\x93\xdes\x00\xe8cY<\xde1.\x19\xd8\xe5R'), (b'\xack\x036\x9b4\xca\xa4\x15\xbd\xd5\xbe\xd3\xd4\xfb{', b'm?\xc44>\xf4qig\x10\x8c\xeb\x11o\x99\xb4'), (b'D!\x0e*\xb3/<\x15F_\xd6;j\x85\xbe,', b'Y\xb01H\xb6:\x04_]OL\x9d\x8e\xbcp\xdc'), (b'\xe6\x9d\x02L[\x1c\xee\x8b0lu\xdd\x00\x9a\xfa!', b'J&\tkf\xa8\xeb\x1b\x90\x81\xce\\\x1b\xe8\xb6\x1d'), (b'\xcdnd\xf6\xf6\x11\x9e\x06\x10\xce\xcfP\x89\xed\xad\x02', b'I:\xe0{8H\xfcw\xb1\xe9v\x11p-/0'), (b'\xa22?\xcd\xfbU+\xbf- \xe3<\xc2\x94\xef\xb6', b'\xc7\x89\xc4K?{\x18\x1ftd\x0f\x19L}\xba\xf9'), (b'8\xf9\xe9\xd0\xa0\x97\x8e\rp\xe9\xd6[\x8e`\x93\x81', b'\xf7\x16"\xaa\x86\\b\x9d\x0c\x17"\x81\xeea\x03\x19'), (b')\xfb\x0c\xc7-D\xd8\xa4\xf7\x8d%\x02\x06\xa1\xbf\x94', b'\xc8s!s\xe2n\xee\xb0\x1eC\x15\xc0\xb4d\x08\xa7'), (b'\x7f\xa6\xdc\xf4\x9e\xe3\xda\xd3\x8a\xcb\xfc\x98,1\\\xe5', b'\x82,\xfbKL_\xe9\xc9L`\xc7"&3#\xb5'), (b'&\x98,\x88\xff\x0e\xca[\xcb\xc6t;\xbc5Y"', b'\x1b\x8bA\xd1\x8a\x9e\xe6\xce\xc4#C\x89\xe9Uon'), (b'\x1c\x1b\xac2\x86\xd6\x9a\xf9\xdd\x8d=\xa3\xff\x9a\xe1\xa5', b'\xed\x10\xed\xc1\xfe\xec\xd3\x0b\x8b\xe9\xd4?H"\xd6\x8f'), (b'9\x8f3\x8b\x02\xb7\xe7\xe5\xa1TO\x1a\xb6N\x0b\xb6', b'\xb4\x0f\xa7 \xc7\x83\x81\x18\x9cU\xfd2\x93\xb8(\xe9'), (b'\x8d\xbe\xcf`\xc3d\xde\xca\xf4\n.\x92Mrw\xa6', b';\t\xfb\x19eKt0\x8foG\xcd\xd5\xe0\xf6\x9d'), (b'd\xbcG)\xec\xc8\x04 \xdd\xd9\xf6\xd6\x91/\xb2\xd3', b'2\xba\xa8\x8a\xc5\xd6h\xab\x9c\x9e\x8b\x1b5\xcc_.'), (b'\x98\xea\xf0\xcbJ-c\xea\x81\xcf?B\x17\xcb\x99\xf9', b'\x82\x05\x14\xf2\x05q\x97\xf7Z-B\xad\xa2\x16\xa0`'), (b'\x1c\x94A\xfd\xe1\xf2\x06\x8b\x19\x80\xfa\x01g\xc3V\xaf', b'\x1c\xde\xec\xdf&\x93u\xab\xf9\x12\xfb\x04\xbc0\x99d'), (b'6\xfa\x993b\t\x12E\xd2t\xde\xb6\xfa\x98Im', b',\xa2\xf7VB\x81\x048\xc8\xbf3\xce\x8d\xecn\x82'), (b'L\xd6\x93\xbe\x02\xbeT\xa1\x8d\x88\x84\x18\xc7\xcf\x9fE', b'\x16i"\xb7\x81\xef7\x7f\xdb\x7fY\xbft\x1b\xac\xd0'), (b'\x90\xd6\xe7>Z\x8b\x83\x14\xab\xefH\xa1\x95\xe6\x8e\xc3', b'\xebJ\xfcXd\xd75I@\x12\xe9\x88\xdcG\x15\xb7'), (b'\xb8\xd6\xd03j\xed\xf9\xa0\xb1L3\x8d\x89P\xb4\xe5', b'\xe3\xdf)CR\x03s\x19\x8f\xb5\xc6\xf5\xee&W\xee'), (b'cx\xd7dQ\xc3)YR\x9c7\xf0\xb4\xe4P\xfd', b'h\xa8\x122\xceV\x8c\xf4\x12\x14J\x08\x91\xcf\x93\xfa'), (b'\x1d\x14\x86\xccTi\xdd\xc3L!\x97\x92\xb3\xb7X$', b't\x84\xa4\xe72\xa5X\xe9Ta\x03\x0c\xb00\x07\x83'), (b'\xa6\xdd\x1f\xfc\xca2\x88\x99\xbeA\x9a\xa6JVJ\x84', b'B\x81\x8e-K\xde\x15\xfd\t"\x08\x04@/-\xc9'), (b']b\x18\xe65C\x17*\x103J?\xe79!\x9b', b'\n\xbb\x1f\x98\nZZ\x97\xe0\x12\xf6W\xc4\x1e_~'), (b'\x9a\x9f-c\t\x14\xa4NL\xc5\xc0\x99R\x15\x94\xca', b'\xa0\xf9\x11\xfa\xd7\xf3\xb5\x11\xb2SK\xe8\xdb\x06\x94\x1f'), (b'!\xf0\x90\xd8t_O\xc1\xb5\xfe}\xca\xa6\x15}\x19', b'\xc9+\x89a9y3\xe0\xffh\xedmMq\x13\xcc'), (b'`\xf1\xab\x07A\xe9\xe5\x805\xf2\xabD8\xce\x85A', b'\x9d\x0f\x91\xc2\x01\x93\x8fm\xd6\x03XZl\t\x15J'), (b')\r\xcf\xc0\xa1\xa9\xf7i\x94\xa9\x8d\xed\xea\xcd\xb0B', b'jq\xe9\x01*H\x14\xa7M\x8c\xd2\xa5\x19\\\x80\x12'), (b'vJ\x98\xbf\xd5D\x15\xdb\xa1\xd7\xdc\xaf\xa0\xaa\xf7\xc1', b'\x06\xd8-\x05\xfe\x172W\xa4\xab\xe4\x97_h\xe6\xac'), (b"\xc6'P\xd2\xfcq\xcb\n\xe2\xc1sN>\xbc\x9a\x08", b'\xacysx4z\x96\x8f\xf1\xa5M\x9e\xe7f\\\x16'), (b'\x00\xc6\x81\xf0\xf1\xcb\xa8_\x1fF\x18\xe9\xb0\x95\xa3\xab', b'N\xd8\xdaz\xe8F4\xb1\x88>\xec\xd6[\x15\xd7\xfd'), (b'E\x1ee\x8f\xd5\x0f\x19\xa8\xb6~-\xe0n\xe7\x89\xe1', b'\x83\x90\xa0\x93\xec\x086+\xcf\x08\x9c\x048\x85\x88^'), (b"\x82\x13\xfa\x9f 6\xd3'\xd7\xd2\xc1\xf9\x03.\x13*", b'J\xe2\xf8\x1e\xec\xeb\xd9"q\xbb:\x04\xc2\xdf<\xa8'), (b'\x01\xdf\xfcPW\xc80&{\xfe\x99\xf8\x10\xc6R\xce', b'=Cnh3\x9drKE\x1aU\x8e\xcc\x84jS'), (b'<\xeeo[J\xb0r0N\x07\xf4\x1e\xfb\x16\xac\x07', b'\xf9KP\xfe\xd2\xed\xc7`\xc7\x0c\x9et\xd3^\xc9g'), (b'P\xa1b#\x07V\xb0s\x86S\x9a\x1d[\xd9\xe7\x92', b'r&\xb1a\x0ff2\xd2\x8ft\x13\xff\xba\x0fy\xf4'), (b'\xc8t\xcc\x96V\x8bS\xde\x98\xf1\xa33\xa9\xdb\x1b\x85', b'13\xf1&\x8c\xfe3\xf9\xca\\\xc6\xdb\x1dT\x04b'), (b'\xb9u\xf2O\xfb\x05\xb3\xc2r*\x17d\\\x8f9M', b'\xeeQ\xf9&\xc5\xc8\xc8\xe6B\x1d\xcf\x184\x19]b'), (b'\x96\xe6\xc9\x1e;\x83e4\xb0\xbe\xf6\x94\x03\x06\x85:', b'\xa2\xb0\nS\xaf\xads.\x94\x16s\x901\xaf-\xe4'), (b'&\x81t\x048(\xd7 \xb6\x1aR\xaa\x98\xdbN\x0b', b'\x9cd\x8a#\xdd \xdcS\x14\x1e\x1b\xea\x87\x1b\xa41'), (b"\x1fC\x0cr,\x1d\xab\xa8\xd9\xfc\xa1\xa2\xbb\xcdw'", b'D\xb6)\x7f\xd7\xe6\xb2@\x18D\x1d\x08c\xce2k'), (b'$\xc1\xabD\r\xe0a\x0co\x12\x17\xbdB\x06\xc9y', b's\xd8\xf8\x05\x81\xfeY\xc4\xc8|\xe5\xe2]U\xfa"'), (b'\x8f\xe3\xd6\xc9d\xc4\xf0\xd0\xe3\x17\x12\x82\xb2\xfd\xe9\xef', b'\xf1\r/9\x97/\xd4\xf6dz\x89\x7fEG/\x15'), (b'\x16\xf2\xd8\x9a\x029\x9b{\x02\xf4u\x08~\x80\xb9Z', b'\xe1+:9\xaa\x87:\xf0\x02\xdd\xe5}G\x0ch\x98'), (b'\xe5\n\xf8gP\xfd\x8aT\x04\xf2\xdc\xa4dgcO', b'\x13\xd0\x1c[\xe6\x8aa\xd6\xa0a\xb1X\x84\xb8\xde2'), (b'\xa8\xaa\xa8\xe7\xcal>\xe6\xdb\x18vL\xdak%}', b"\xe5W'\x83\xbd\xf2\x01\x9b\x89\x98l\xfd\xcf\x02\x84\x13"), (b'6\x9b\xae\x07\xe0+\xa0\xcd\xdbT\t\x02@\x81$E', b'\x10Q\x1an\xaa\xe7\x9cy\x04t\x16\x1d\xea\x9d\xd2\xe6'), (b'ne\x83\xb0\xe2\x8aG\xda\xaf%\x93cg\x84\xaf\xea', b'p!}\x8aQ(C\x08\xaf\x8bi\x80\xdaw\x7f\x81'), (b'<\xa6\x90\xb2\x03\x8f\xa4\x17$\xc7\xbe\x8a\x1e!\x01s', b'\x19\x7ftaaLUB\x1aBt\xf5U~\xeeY'), (b'\xe4|z\xaer8\xa0\xa3\xc8\x9c\x8d\x03`\xb8rN', b'b\xda\xf8e\n\x1f\xc1\xb2\x97\x13\xd7\x91\x91\x86\xbe\x13'), (b'\x00\x84\xc1\xd8\xe0ub\x1a\xc0\xa1\xfe\x08K\xfb>,', b'g\xb06$\xbb{s\xb9-\x8e5h|+\xf8\xb1'), (b'(\xac{\xb1^OOy\xc98\xf5\xc5\xc2\xd3\xb3\x1e', b'\x8d\xfc\x0f\xda\xd5\xa6\x89)\x92\x173\x1e)*\xf4&'), (b'G*.\xe2\xd8\xee\x87\xfa\x83~eS\xcc\x1e\xdeQ', b'\xc9y \x1a]Y\x8d\xf8\xa0\xd0^p\xc1\xf1\xa4\xd2'), (b'\x16\x91\xd7[\xbeC\x80T\x08\x0f\xbd\xe1\xf7\xbd\x1b\xb8', b'\xcc\x8c\xfb\x1f"\x16\xa7/\xe2inK\xaa\xdf\xc0\xe4'), (b'\x1d\xbc\xf2\x9b\xe90\x13\xc9\tuuk\r\xe4\xa9\xfa', b'\xf6\x97\xf2B\xaa\x83\xf9k\x15K\xa0\xb9\xdf\xf4,\x02'), (b'hIA(;\x8b\x92G!K\xca\x1a\xfd\x8c9\x95', b'0!V\xfc\xfb\xa1,R\xc6\xdaO\x9c\x16\xa8<\xe2'), (b'?\xd8\xd7e\x88\xad\xcd\xd6\x8d\\\x1e\xc2l$;\xd8', b"\x88\x8e\x86'#f\xcf\xe3,\x96&\r\xb1\xee@g"), (b"\x81\n\x18\xd7\x92\\dv'\x1a\xea\x8c7\xf2S\xb0", b'\x99\xbf\xf9qo\xde\x92xge\x04\x84\xb8 \xfb\xe3'), (b'\xa7\x8d\x83\xc8\xa3\xc15\xdc`\x82\xed\xae.\xea\xf2\xea', b'[\xe8{\r\xd6g\xcf\xe8L\x16\xde\xcd\x90M\xbd\x9e'), (b'\x1a\x1b;\xa62\x10i\xb2\x19n\xc1\xddKz\xb4\xd8', b';On\xd2N\xef%B\x05_T9)IG\xcf'), (b'@g`\x1e\xc1\xdf\x14A\xc1\x04N\xb6]\x97\xd3\x17', b'\xbc6\xcb\xaah*\xf1\xd0_\xd9B\x94F\xa8\xd8\r'), (b'<\x86\xa4z\xb4\xaeS\x90\xa9\xb5I\xe8\xe6FI\x8c', b'\r2\xd2.H"\xfb\x9d}{\x11^\xdf\xcb\x186'), (b'\xf0\xb8J|\x0b\xfe\xcd\x8b\xc0$&"8\xffob', b'\xa8}\xf31\xdd\x07\xc6\xa0\xbb\x9f0\xca\xe3&^\xf6'), (b'r\x9b\x81\xa0D\x94\x80\xfdu\xb1O\xbc\xf6\x1a\xd8\xe0', b'\xd4)\xb6\xdbQ\xa5#\x16\xed9\xadM\xf82%\xeb'), (b'\xb9V\xb7R\xb9\xcdJ9c\x1a`V\xb9\xb0oX', b'\x9a\xff]\xb3\x83\x96\xf5\r\x98\xe2\xc5\x86\xa21k%'), (b"J\xe9\xaa\x8f\x07\xab\x90\x18'\xa6\x85\x08\x18\xa8\x96?", b'\xa7\x83\xed\xd9\t\xd8\x85\xe4\xf8\xcb0\x95n\xf2`\xa8'), (b'\xc3\xaf\xc8m\xfc\x90}=\xb8\x1fr\xab}\r\xc1!', b'\x04\xa0sq\xf7\xd3Dm\xb9\xb8\x9c\xbap(\xa1L'), (b'\x97bq\x0f\x93\xb5\xa1\xa1\xf58\xdcu5\x9a\xbd\x8b', b"\x07'\xc0\xdb/0\xd9H\x81\x06\xe2^ `F\xa9"), (b'Nn\xd4\xbd\x13dN\x9e\xcd\xd4\x06kk~\xf5\x7f', b"\x98j'}\xc5\xb0\x1d\x19\xe2/\x92'\x9e\xd4\xba("), (b'\x08ty\x84\xde>\x0f5jL6\x05\xe1\x02pH', b'0\xb9\xd5\xd40u\x04\xfdK\xe2D\xbc\xddy\xd4\x11')]
def cost(func):
    def wrapper(*args,**kwargs):
        start=time.time()
        res=func(*args,**kwargs)
        end=time.time()
        spend=end-start
        print(f"cost:{spend}")
        return res
    return wrapper
@cost
def test(repeat):
    tmp=[*lis*repeat]
    if example.get_list_size()<4000000:

        for it in tmp:
            t=b'something here'
            example.init_map(it[0],t ,16,len(t),int(random.random()*10000))
 
    print(f"pool size is {example.get_list_size()}")
    print(f"get args from lis start")
    print(lis)
    arg1=lis[0][0]
    print(f"get args from lis complete")
    t=b'something need search'
    print(f"start search")
    res=example.search(arg1,t,16,len(t),0.1,int(random.random()*10000))
    l=len(gc.get_objects())
    print(f"res is {len(res)},gc:{l}")


if __name__=='__main__':
    for i in range(10000):
        test(10)

error

pool size is 1000
get args from lis start
Fatal Python error: Segmentation fault

Current thread 0x00007f0ca3c90340 (most recent call first):
  File "/tmp/tmp.ztKz0S1yKb/test.py", line 29 in test
  File "/tmp/tmp.ztKz0S1yKb/test.py", line 12 in wrapper
  File "/tmp/tmp.ztKz0S1yKb/test.py", line 41 in <module>
Segmentation fault (core dumped)

I tried using PyMem_RawMalloc and Py_DecRef but still segfault. what should i do?

Sorry for the little misinformation I gave. I may have located where the problem is: problem here

Everything works when I comment out Py_DecRef ,otherwise it throws a seg fault exception:

gdb backtrace,
print info

I don't understand why Py_DecRef can't be used here.

Solution

  • There are some things wrong:

    Before going further, I suggest a thorough reading of the following (including referenced URLs):

    Now, there are multiple problems, each with multiple solutions.
    Starting from your code, I prepared an example. It's written in C (lots of code which doesn't have anything to do with the goal).
    All the business logic is carried out in C (it's callable from a C application), and the Python wrapping layer is on top of that.

    Outputs: