python-3.xpandasnumpygloblowpass-filter

Reading in multiple files in Python and saving them one by one in a different directory


    import glob
    import pandas as pd
    import seaborn as sns
    import numpy as np
    from scipy import signal
    import matplotlib.pyplot as plt
    
    files = glob.glob("Angular_position_*_*.csv")
    output = pd.DataFrame()
    
    for f in files:
        df = pd.read_csv(f)
        time = df.iloc[:,0]
        time = time.to_numpy()
        ynew = df.iloc[:,1:]
        ynew = ynew.to_numpy()
        
    
        lowPassCutoffFreq = 6.0 # Cut off frequency
        Sample_freq = 150; #Target sample frequency
        N = 2 # Order of the filter; In this case 2nd order
        Wn = lowPassCutoffFreq/(Sample_freq/2)  #Normalize frequency
    
        b, a = signal.butter(5, Wn, btype='low',analog=False,output='ba')
        #scipy.signal.butter(N, Wn, btype='low', analog=False, output='ba', fs=None)
    
        output = signal.filtfilt(b, a, ynew, axis=0)
        
     
        np.savetxt("enter directory path/Filtered_files/Filtered_Angular_position_*_*", output,   delimiter = ', ', newline = "\n")

I am trying to read in all files in a directory, they are then low pass filtered. After that the results are saved one after the other but not in one file. The result gives each files with 3 columns and ideally I would like them to named with headers e.g. col1, col2, col3.

Without using glob, I can filter all my files individually but I have more than 100 such files.

Any help would be appreciated.

best wishes,


Solution

  • I have partially solved the issue apart from the header names:

    import glob
    import pandas as pd
    from tnorma import tnorma
    import seaborn as sns
    import numpy as np
    from scipy import signal
    import matplotlib.pyplot as plt
    
    path = r'location_of_dir'
    
    all_files = glob.glob(path + '/*.csv')
    # yn = np.zeros(shape = (101,1))
    # tn = np.zeros(shape = (101,1))
    #ynew = []
    
    yn = np.zeros(shape = (101,1))
    
    for filename in all_files:
        df = pd.read_csv(filename, index_col=None, header=0)
        print(filename)
        foo = filename.split("/")[-1]
        #df = pd.read_csv(f)
        time = df.iloc[:,0]
        time = time.to_numpy()
        ynew = df.iloc[:,1:]
        ynew = ynew.to_numpy()
        #print(ynew)
    
        lowPassCutoffFreq = 6.0 # Cut off frequency
        Sample_freq = 150; #Target sample frequency
        N = 2 # Order of the filter; In this case 2nd order
        Wn = lowPassCutoffFreq/(Sample_freq/2)  #Normalize frequency
    
        b, a = signal.butter(5, Wn, btype='low',analog=False,output='ba')
        #scipy.signal.butter(N, Wn, btype='low', analog=False, output='ba', fs=None)
    
        output = signal.filtfilt(b, a, ynew, axis=0)
        #print (output)
        tn = np.linspace(0, 100, 101)     # new time vector for the new time-normalized data
        yn, tn, indie = tnorma(output, k=3, smooth =1, mask = None, show = False)
        
        np.savetxt("path_name/foldername/file"+ foo, yn, delimiter = ', ', newline = "\n")
    

    However, I am having difficulty in putting header names on the 3 columns per file.