Переходи на ViewPager2 и ставь viewPager.isUserInputEnabled = false или
viewPager.addOnPageChangeListener(object : ViewPager.OnPageChangeListener {
override fun onPageScrolled(position: Int, positionOffset: Float, positionOffsetPixels: Int) {}
override fun onPageScrollStateChanged(state: Int) {}
override fun onPageSelected(position: Int) {}
})