RecyclerView scrolls to top when dragging item with ItemTouchHelper

I have this weird bug where my RecyclerView scrolls back to top position whenever I start dragging an item in it. It's inside ViewPager if that has any difference. You can see the behavior in .gif attached.

EDIT:

It seems that RecyclerView view scrolls to top when notifyItemMoved is called and it scrolls just as much for the first view to be at least partially displayed on screen.

View

<androidx.recyclerview.widget.RecyclerView
    android:id="@+id/accounts_recycler_view"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:scrollbarStyle="outsideOverlay"
    android:scrollbars="none"
    app:layoutManager="androidx.recyclerview.widget.LinearLayoutManager"
    tools:listitem="@layout/view_account_list_item" />

Adapter

class AccountListAdapter(
private val onAccountClickListener: OnAccountClickListener) :
ListAdapter<Account, AccountListAdapter.ViewHolder>(
    AccountDiffCallback()
) {

override fun onCreateViewHolder(parent: ViewGroup, viewType: Int): ViewHolder {
    val inflater = LayoutInflater.from(parent.context)
    return ViewHolder(
        inflater.inflate(
            R.layout.view_account_list_item,
            parent,
            false
        )
    )
}

override fun getItemId(position: Int): Long {
    return getItem(position).accountId.toLong()
}

override fun onBindViewHolder(holder: ViewHolder, position: Int) {
    holder.bind(getItem(position), onAccountClickListener)
}

class ViewHolder(itemView: View) : RecyclerView.ViewHolder(itemView), OnItemDragged {

    fun bind(account: Account, onAccountClickListener: OnAccountClickListener) {
        itemView.account_name.text = account.name
       
        itemView.setOnClickListener {
            onAccountClickListener.onAccountClick(account)
        }

    }

    override fun onItemSelected() {
        itemView.setBackgroundColor(
            ContextCompat.getColor(
                itemView.context,
                R.color.background_contrast
            )
        )
    }

    override fun onItemClear() {
        itemView.setBackgroundColor(
            ContextCompat.getColor(
                itemView.context,
                R.color.background
            )
        )
    }

}

class AccountDiffCallback : DiffUtil.ItemCallback<Account>() {
    override fun areItemsTheSame(oldItem: Account, newItem: Account): Boolean {
        return oldItem.accountId == newItem.accountId
    }

    override fun areContentsTheSame(oldItem: Account, newItem: Account): Boolean {
        return (oldItem.balance == newItem.balance
                && oldItem.annualReturn == newItem.annualReturn
                && oldItem.name == newItem.name)
    }
}

interface OnAccountClickListener {
    fun onAccountClick(account: Account)
}

interface OnItemDragged {
    fun onItemSelected()
    fun onItemClear()
}}

ItemTouchHelper

 private fun setupListAdapter() {

    accountListAdapter = AccountListAdapter(this)
    accountListAdapter.setHasStableIds(true)

    accounts_recycler_view.adapter = accountListAdapter
    accounts_recycler_view.addItemDecoration(
        DividerItemDecoration(
            requireContext(),
            DividerItemDecoration.VERTICAL
        )
    )


    val accountTouchHelper = ItemTouchHelper(
        object : ItemTouchHelper.SimpleCallback(
            ItemTouchHelper.UP or ItemTouchHelper.DOWN,
            0
        ) {

            override fun onSelectedChanged(
                viewHolder: RecyclerView.ViewHolder?,
                actionState: Int
            ) {
                if (actionState == ItemTouchHelper.ACTION_STATE_DRAG) {
                    val accountViewHolder = viewHolder as AccountListAdapter.ViewHolder
                    accountViewHolder.onItemSelected()
                }
                super.onSelectedChanged(viewHolder, actionState)
            }

            override fun clearView(
                recyclerView: RecyclerView,
                viewHolder: RecyclerView.ViewHolder
            ) {
                val accountViewHolder = viewHolder as AccountListAdapter.ViewHolder
                accountViewHolder.onItemClear()
                super.clearView(recyclerView, viewHolder)
            }


            override fun onMove(
                recyclerView: RecyclerView,
                viewHolder: RecyclerView.ViewHolder,
                target: RecyclerView.ViewHolder
            ): Boolean {
                val fromPos: Int = viewHolder.adapterPosition
                val toPos: Int = target.adapterPosition
                Collections.swap(_accountList, fromPos, toPos)
                accountListAdapter.notifyItemMoved(fromPos, toPos)
                return true
            }

            override fun onSwiped(viewHolder: RecyclerView.ViewHolder, direction: Int) {}

        })

    accountTouchHelper.attachToRecyclerView(accounts_recycler_view)

}

So the issues was that my Recyclerview was inside ViewPager which was inside a ConstraintLayout . View pager was constrained vertically with height set to 0dp but width was set to match_parent. All I needed was to constrain it horizontally with width set to 0dp and setHasFixedSize = true to RecyclerView

When calling notifyItemMoved for adapter, if RecyclerView is flexible, all items are redrawn and by default it focuses on the first item.