Can I make a WPF ListBox change selection on left button press only

I have a WPF ListBox operating in single selection mode. I am adding drag and drop to move items around. Currently the ListBox selection responds to both left button pressed and then with mouse moves with left button down. So after I wait for the MinimumVerticalDragDistance to start a drag operation, a different item could be selected. Dragging either the unselected orginal item or dragging the new selected item is confusing. Adding 'e.Handled=true' in xxx_MouseMove or xxx_PreviewMouseMove does not do anything. Any ideas on suppressing this selection due to mouse moves with left button down?

The best kludge I came up with is to cancel the ListBox's "Selection by dragging" in the IsMouseCapturedChanged event.

    public partial class MainWindow : Window
{
    Rect? dragSourceGestureRect;
    bool busy;

    public MainWindow()
    {
        InitializeComponent();

        listBox.ItemsSource = Enumerable.Range(1, 9);

        listBox.PreviewMouseLeftButtonDown += listBox_PreviewMouseLeftButtonDown;
        listBox.IsMouseCapturedChanged += listBox_IsMouseCapturedChanged;
        listBox.MouseMove += listBox_MouseMove;
    }

    void listBox_IsMouseCapturedChanged(object sender, DependencyPropertyChangedEventArgs e)
    {
        if (busy)
            return;

        if (!listBox.IsMouseCaptured)
            dragSourceGestureRect = null;

        else if (dragSourceGestureRect.HasValue)
        {
            busy = true;
            {
                //tell the ListBox to cancel it's "Selection by dragging"
                listBox.ReleaseMouseCapture();
                //Now recapture the mouse for canceling my dragging
                listBox.CaptureMouse();
            }
            busy = false;
        }
    }

    void listBox_PreviewMouseLeftButtonDown(object sender, MouseButtonEventArgs e)
    {
        var center = e.GetPosition(listBox);
        dragSourceGestureRect = new Rect(
            center.X - SystemParameters.MinimumHorizontalDragDistance / 2,
            center.Y - SystemParameters.MinimumVerticalDragDistance / 2,
            SystemParameters.MinimumHorizontalDragDistance,
            SystemParameters.MinimumVerticalDragDistance);
    }

    void listBox_MouseMove(object sender, MouseEventArgs e)
    {
        if (!dragSourceGestureRect.HasValue || dragSourceGestureRect.Value.Contains(e.GetPosition(listBox)))
            return;

        dragSourceGestureRect = null;

        var data = new DataObject(DataFormats.UnicodeText, "The Data");
        DragDrop.DoDragDrop(listBox, data, DragDropEffects.Copy);

        e.Handled = true;
    }
}