Avoid SwipeRefresh while fastScrolling in android

I have a ListView having fastScroll always enabled and SwipeRefresh implementation. When i swipe the list downward, it refreshes the list. My problem is fastScroll. If the list has its first item visible or say at its initial top, then if scroll the fastScrollThumb downward, it does the Swipe effect not the scroll down. Is there anyway/solution that if i press the fastScrollThumb , then it should not do Swipe refresh effect rather it should scroll down as it natural behavior.

EDITED My XML Layout is as follow:

    <?xml version="1.0" encoding="utf-8"?>
    <android.support.v4.widget.SwipeRefreshLayout 
    xmlns:android="http://schemas.android.com/apk/res/android"
    android:id="@+id/swipe_container"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:background="@color/white" >

    <RelativeLayout
        android:layout_width="match_parent"
        android:layout_height="match_parent" >

        <RelativeLayout
            android:id="@+id/buttons_layout"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_centerHorizontal="true"
            android:layout_marginTop="10dp"> 

            <ImageView
                android:id="@+id/SubmitButton"
                android:layout_width="wrap_content"
                android:layout_height="wrap_content"
                android:background="@drawable/neoo_tab_selector" />

        </RelativeLayout>


        <ListView
            android:id="@id/android:list"
            style="@style/NeeoContactListView"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            android:layout_below="@+id/buttons_layout"
            android:layout_marginTop="10dp" />
    </RelativeLayout>

</android.support.v4.widget.SwipeRefreshLayout>

My Logic for onScroll for enabling/disabling the SwipeRefresh is :

    getListView().setOnScrollListener(new OnScrollListener() {

    @Override
    public void onScrollStateChanged(AbsListView view, int scrollState) {
    }

    @Override
    public void onScroll(AbsListView view, int firstVisibleItem, int visibleItemCount, int totalItemCount) {
        boolean enable = false;
        if(getListView() != null && getListView().getChildCount() > 0){
            // check if the first item of the list is visible
            boolean firstItemVisible = getListView().getFirstVisiblePosition() == 0;
            // check if the top of the first item is visible
            boolean topOfFirstItemVisible = getListView().getChildAt(0).getTop() == 0;
            // enabling or disabling the refresh layout
            enable = firstItemVisible && topOfFirstItemVisible;
        }
        if(enable){
            enableSwipe();

        }else{
            disableSwipe();

        }
    }
});

I just looked into the documentation of the SwipeRefreshLayout and I think they mention what you are looking for:

... If the listener determines there should not be a refresh, it must call setRefreshing(false) to cancel any visual indication of a refresh. If an activity wishes to show just the progress animation, it should call setRefreshing(true). To disable the gesture and progress animation, call setEnabled(false) on the view.

So you have a few options, I would first try playing around with setEnabled() or setRefreshing(), but to properly use those methods we first need to be able to detect if the ListView is fast scrolling. There is no listener or anything for fast scrolling but you can get the state through reflection! Based on this excellent and seriously underrated answer I have written a FastScrollListener which can be used just like a OnScrollListener:

this.listView.setFastScrollEnabled(true);
this.listView.setOnScrollListener(new FastScrollListener(this.listView) {

    @Override
    protected void onFastScrollStateChanged(AbsListView view, FastScrollState state) {
        // This line disabled the SwipeRefreshLayout  
        // whenever the user is fast scrolling
        swipeRefreshLayout.setEnabled(state != FastScrollState.DRAGGING);
    }

    @Override
    protected void onFastScroll(AbsListView view, int firstVisibleItem, int visibleItemCount, int totalItemCount) {

    }
});

You could also try calling setRefreshing() instead of setEnabled() if setEnabled() doesn't work.

And here is the code of the FastScrollListener:

public abstract class FastScrollListener implements AbsListView.OnScrollListener {

    private final int STATE_DRAGGING;
    private final int STATE_VISIBLE;
    private final int STATE_NONE;

    public enum FastScrollState {
        DRAGGING,
        VISIBLE,
        NONE,
        UNKNOWN
    }

    private FastScrollState fastScrollState = FastScrollState.UNKNOWN;
    private Field stateField;
    private Object mFastScroller;

    public FastScrollListener(AbsListView listView) {
        try {
            final Field fastScrollerField = AbsListView.class.getDeclaredField("mFastScroller");
            fastScrollerField.setAccessible(true);
            mFastScroller = fastScrollerField.get(listView);

            final Field stateDraggingField = mFastScroller.getClass().getDeclaredField("STATE_DRAGGING");
            stateDraggingField.setAccessible(true);
            STATE_DRAGGING = stateDraggingField.getInt(mFastScroller);

            final Field stateVisibleField = mFastScroller.getClass().getDeclaredField("STATE_VISIBLE");
            stateVisibleField.setAccessible(true);
            STATE_VISIBLE = stateVisibleField.getInt(mFastScroller);

            final Field stateNoneField = mFastScroller.getClass().getDeclaredField("STATE_NONE");
            stateNoneField.setAccessible(true);
            STATE_NONE = stateNoneField.getInt(mFastScroller);

            stateField = mFastScroller.getClass().getDeclaredField("mState");
            stateField.setAccessible(true);
            fastScrollState = getFastScrollState();

        } catch (NoSuchFieldException e) {
            throw new IllegalStateException("Could not find required fields for fast scroll detection!", e);
        } catch (IllegalAccessException e) {
            throw new IllegalStateException("Could not find required fields for fast scroll detection!", e);
        }
    }

    @Override
    public void onScrollStateChanged(AbsListView view, int scrollState) {
        updateFastScrollState(view);
    }

    @Override
    public void onScroll(AbsListView view, int firstVisibleItem, int visibleItemCount, int totalItemCount) {
        updateFastScrollState(view);
        updateFastScroll(view, firstVisibleItem, visibleItemCount, totalItemCount);
    }

    private void updateFastScrollState(AbsListView view) {
        if (stateField != null) {
            final FastScrollState state = getFastScrollState();
            if (fastScrollState != state) {
                fastScrollState = state;
                onFastScrollStateChanged(view, state);
            }
        }
    }

    private void updateFastScroll(AbsListView view, int firstVisibleItem, int visibleItemCount, int totalItemCount) {
        if(fastScrollState == FastScrollState.DRAGGING) {
            onFastScroll(view, firstVisibleItem, visibleItemCount, totalItemCount);
        }
    }

    private FastScrollState getFastScrollState() {

        try {
            final int state = stateField.getInt(mFastScroller);

            if (state == STATE_DRAGGING) {
                return FastScrollState.DRAGGING;
            }

            if (state == STATE_VISIBLE) {
                return FastScrollState.VISIBLE;
            }

            if (state == STATE_NONE) {
                return FastScrollState.NONE;
            }

            return FastScrollState.UNKNOWN;
        } catch (IllegalAccessException e) {
            throw new IllegalStateException("Could not read fast scroll state!", e);
        }
    }

    protected abstract void onFastScrollStateChanged(AbsListView view, FastScrollState state);

    protected abstract void onFastScroll(AbsListView view, int firstVisibleItem, int visibleItemCount, int totalItemCount);
}

I hope I could help you and if you have any further questions please feel free to ask!

---

EDIT: Try this, but I doubt it will work:

@Override
protected void onFastScrollStateChanged(AbsListView view, FastScrollState state) {
    boolean enable = false;
    if (getListView().getChildCount() > 0) {
        boolean firstItemVisible = getListView().getFirstVisiblePosition() == 0;
        boolean topOfFirstItemVisible = getListView().getChildAt(0).getTop() == 0;
        enable = firstItemVisible && topOfFirstItemVisible;
    }
    refreshLayout.setEnabled(enable);
}