StaggeredGridLayout manager after deleting item, items gets scattered

I am using a Staggered Grid Layout to show data to recycler view from database.I have faced a problem where after deleting an item from db as well as remove position from adapter, I got some item rendering issues.Like the scattered all over the place. Here is my code

 @Override
protected void onCreate(Bundle savedInstanceState) {
    super.onCreate(savedInstanceState);
    setContentView(R.layout.activity_note_favorite);
    ButterKnife.bind(this);
    toolbar = (Toolbar) findViewById(R.id.fav_note_toolbar);
    setSupportActionBar(toolbar);

    noteAdapter = new NoteAdapter(noteModelList, NoteFavoriteActivity.this);
    layoutManager = new StaggeredGridLayoutManager(2, StaggeredGridLayoutManager.VERTICAL);
    recyclerView.setLayoutManager(layoutManager);
    recyclerView.setItemAnimator(new DefaultItemAnimator());
    recyclerView.setAdapter(noteAdapter);
    touchListenerRecycler();
    loadDataAll();


}
private void loadAll() {
    noteModelList.clear();

    DBManagerFav dbManagerFav = DBManagerFav.getInstance(NoteFavoriteActivity.this);
    dbManagerFav.openDataBase();
    noteModelList = dbManagerFav.getAllNoteList();
    Log.i(TAG, " size : " + noteModelList.size());
    noteAdapter = new NoteAdapter(noteModelList, NoteFavoriteActivity.this);
    recyclerView.setAdapter(noteAdapter);
    noteAdapter.notifyDataSetChanged();

    dbManagerFav.closeDataBase();

}


 private void deleteOperation() {

    DBManagerFav dbManagerFav = DBManagerFav.getInstance(NoteFavoriteActivity.this);
    dbManagerFav.openDataBase();

    NoteModel noteModel = new NoteModel();
    noteModel.setId(noteModelList.get(adapterClickedPosition).getId());

    int status = dbManagerFav.deleteNote(noteModelList.get(adapterClickedPosition).getId());
    if (status > 0) {

        noteAdapter.removeAt(adapterClickedPosition);

    }
    dbManagerFav.closeDataBase();
    loadDataAll();
}
//this belongs to adapter
public void removeAt(int position) {
    Log.d(TAG, " removing at position : " + position);
    noteModelList.remove(position);
    notifyItemRemoved(position);
    notifyItemRangeChanged(position, noteModelList.size());
    notifyDataSetChanged();
}

I have attached two screenshots before and after Before deleting an item

after deleting an item

Can you point me out what else do I need ?

You can put inside loadDataAll(){

    layoutManager = new StaggeredGridLayoutManager(2, StaggeredGridLayoutManager.VERTICAL);
    recyclerView.setLayoutManager(layoutManager);

//........ } Voila..