Analog measurement incorrect on Teensy 2.0++

I have a Joystick wired up to my Teensy 2.0++ and I want to read the analog values from it.

I took this implementation from PJRC:

static uint8_t aref = (1<<REFS0); // default to AREF = Vcc, this is a 5V Vcc Teensy

void analogReference(uint8_t mode)
{
    aref = mode & 0xC0;
}

// Mux input
int16_t adc_read(uint8_t mux)
{
#if defined(__AVR_AT90USB162__)
    return 0;
#else
    uint8_t low;

    ADCSRA = (1<<ADEN) | ADC_PRESCALER;     // enable ADC
    ADCSRB = (1<<ADHSM) | (mux & 0x20);     // high speed mode
    ADMUX = aref | (mux & 0x1F);            // configure mux input
    ADCSRA = (1<<ADEN) | ADC_PRESCALER | (1<<ADSC); // start the conversion
    while (ADCSRA & (1<<ADSC)) ;            // wait for result
    low = ADCL;                 // must read LSB first
    return (ADCH << 8) | low;           // must read MSB only once!
#endif
}

// Arduino compatible pin input
int16_t analogRead(uint8_t pin)
{
#if defined(__AVR_ATmega32U4__)
    static const uint8_t PROGMEM pin_to_mux[] = {
        0x00, 0x01, 0x04, 0x05, 0x06, 0x07,
        0x25, 0x24, 0x23, 0x22, 0x21, 0x20};
    if (pin >= 12) return 0;
    return adc_read(pgm_read_byte(pin_to_mux + pin));
#elif defined(__AVR_AT90USB646__) || defined(__AVR_AT90USB1286__)
    if (pin >= 8) return 0;
    return adc_read(pin);
#else
    return 0;
#endif
}

I have my X and Y pins wired up to F1 and F0, and I want to retrieve values with the following code:

long map(long x, long in_min, long in_max, long out_min, long out_max) // map method shamelessy ripped from Arduino
{
  return (x - in_min) * (out_max - out_min) / (in_max - in_min) + out_min;
}

joy_ly       = map(analogRead(0), 0, 65535, 0, 255);
joy_lx       = map(analogRead(1), 0, 65535, 0, 255);

I measured my Joystick with a multimeter and it works perfectly (around 2.43V on center, 0V on min, and 5V on max), but the center value always ends up being very close to zero.

Is there anything I'm doing wrong?

NOTE: This is an at90usb1286 chip.

The ADC max value is 1024, not 65535.