Unhide or return selected options on multiple select using jquery

Guys I'm creating a code that will remove a selected option on the other dropdown once selected, I used jQuery (jquery-1.10.1.min) but I don't know how will I be able to return the chosen option or number once the user change their mind.

Can you please help me? Thanks in advance.

Here is my script:

HTML

<select name="question_order[]" id="question_order1">
<option value="">None</option>
<option value="01">01</option>
<option value="02">02</option>
<option value="03">03</option>
<option value="04">04</option>
</select>

<select name="question_order[]" id="question_order2">
<option value="">None</option>
<option value="01">01</option>
<option value="02">02</option>
<option value="03">03</option>
<option value="04">04</option>
</select>

<select name="question_order[]" id="question_order3">
<option value="">None</option>
<option value="01">01</option>
<option value="02">02</option>
<option value="03">03</option>
<option value="04">04</option>
</select>

<select name="question_order[]" id="question_order4">
<option value="">None</option>
<option value="01">01</option>
<option value="02">02</option>
<option value="03">03</option>
<option value="04">04</option>
</select>

Javascript

$(window).load(function(){
$('#question_order1').on('change', function() {
   var val=this.value; 
   $("#question_order2").find("option:contains("+val+")").remove();     
   $("#question_order3").find("option:contains("+val+")").remove();
   $("#question_order4").find("option:contains("+val+")").remove();

 });

$('#question_order2').on('change', function() {
  var val=this.value;
   $("#question_order1").find("option:contains("+val+")").remove();
   $("#question_order3").find("option:contains("+val+")").remove();
   $("#question_order4").find("option:contains("+val+")").remove();

});

$('#question_order3').on('change', function() {
  var val=this.value;
   $("#question_order1").find("option:contains("+val+")").remove();
   $("#question_order2").find("option:contains("+val+")").remove();     
   $("#question_order4").find("option:contains("+val+")").remove();   
});


$('#question_order4').on('change', function() {
  var val=this.value;
   $("#question_order1").find("option:contains("+val+")").remove();
   $("#question_order2").find("option:contains("+val+")").remove();     
   $("#question_order3").find("option:contains("+val+")").remove();   
});


});//]]> 

Here's a solution that first stores a copy of the <option> html. On change of a <selecct> it creates an array of all the selected values . It then rebuilds all the other ones from the stored html and filters the values allowed.

$(function () {
    var $quest = $('.question_order');
    /* cache option html */
    var optHtml = $quest.first().html();

    $quest.on('change', function () {
        /* make array of all selected values*/
        var selecteds=$quest.find('option:selected').map(function(){
            if(this.value){
               return this.value 
            }
        }).get();
        /* rebuild the other select elements*/
        $quest.not(this).each(function(){
            var selVal=this.value || '';
            /* create new set of options with filtered values*/ 
            var $opts=$(optHtml).filter(function(){
                return $.inArray(this.value,selecteds) ==-1 ||  this.value ==selVal
            });
            /* replace children*/
           $(this).html($opts).val(selVal);

        });
    });

});

Note that class="question_order" added to each element.

DEMO