Swipe to refresh is not working if LazyGridView has no child in compose

I'm using SwipeRefresh view in compose (from accompanist). I have LazyVerticalGrid inside it, and grid populates paging data from network. If there's no content in paging data, i'm showing empty state view. When there's some data, swipe-to-refresh is working. Issue is, i'm not able to do swipe-to-refesh in LazyVerticalGrid if there's no data, but same is working in LazyColumn (both case has NoContentView shown).

@Composable
fun GridItems(
    searchViewModel: SearchViewModel
) {
    var isRefreshing by remember { mutableStateOf(false) }
    val posts = remember { searchViewModel.posts }.collectAsLazyPagingItems()

    Scaffold(
        topBar = { MyTopBar() }
    ) { innerPadding ->
        SwipeRefresh(
            state = rememberSwipeRefreshState(isRefreshing = isRefreshing),
            onRefresh = { searchViewModel.getPosts() }
        ) {
            Column(modifier = Modifier.padding(innerPadding)) {
                LazyVerticalGrid(
                    cells = GridCells.Fixed(3),
                    modifier = modifier.padding(horizontal = 3.dp)
                ) {
                    items() {
                        MySinglePostItem()
                    }

                    posts.apply {
                        when {
                            //Other loadState here

                            // Show below view if no item is present in paging data 
                            loadState.refresh is LoadState.NotLoading && loadState.append.endOfPaginationReached && posts.itemCount < 1 -> {
                                NoContentView(Modifier.fillMaxSize())
                            }
                        }
                    }
                }
            }
        }
    }
}

@Composable
fun NoContentView(modifier: Modifier = Modifier) {
    Box(
        modifier = modifier,
        contentAlignment = Alignment.Center
    ) {
        Column(
            modifier = Modifier.fillMaxSize(),
            verticalArrangement = Arrangement.Center,
            horizontalAlignment = Alignment.CenterHorizontally
        ) {
            Image(
                modifier = Modifier.size(48.dp),
                painter = painterResource(id = R.drawable.ic_outlined_image_no_content)
            )
            Spacer(modifier = Modifier.height(4.dp))
            Text(
                text = stringResource(id = R.string.empty_posts_message),
                fontSize = 15.sp
            )
        }
    }
}

As Accompanist states in its documentation SwipeRefresh needs some content that is scrollable in order to be able to react to swipe gestures.

So perhaps having too few elements Compose automatically makes a LazyColumn/LazyVerticalGrid non-scrollable in order to optimize things?

My best guess would be to show a Scrollable-Column when you have no items to show and a LazyVerticalGrid otherwise.

SwipeRefresh(...) {
    if (loadState.refresh is LoadState.NotLoading && loadState.append.endOfPaginationReached && posts.itemCount < 1) {
        Column(Modifier.verticalScroll(rememberScrollState())) {
            NoContentView(Modifier.fillMaxSize())
        }
    } else {
        LazyVerticalGrid(...) {
             // ...
        }
    }
}