Getting Pointer or Reference to mapped_type from Arbitrary Map

Edit: I've found and written up a solution to my problem but I've left the question unanswered since my solution may still not be ideal.

I'm writing a small library designed to perform routines on maps of maps, but I'm having trouble designing a set of class templates that will let me get a pointer or reference (depending on the second_type of the map's value_type) to a map's mapped_type, regardless of the type of the map (e.g. std::map, boost::ptr_map).

To elaborate further, I have tabulated some input types and desired output types.

Case   Input Type                                   Output Type
 A     std::map<int, std::map<int, int> >           std::map<int, int>&
 B     std::map<int, boost::ptr_map<int, int> >     boost::ptr_map<int, int>&
 C     boost::ptr_map<int, std::map<int, int> >     std::map<int, int>* const
 D     std::map<int, std::map<int, int> >*          std::map<int, int>&
 E     std::map<int, boost::ptr_map<int, int> >*    boost::ptr_map<int, int>&
 F     boost::ptr_map<int, std::map<int, int> >*    std::map<int, int>* const

My code passes cases A, B, D and E, but fails on cases C and F. Here is what I have so far.

template <class Map>
struct map_utils
{
    template <class K>
    static typename
    boost::remove_pointer<
            typename Map::value_type
    >::type::second_type&
    get(Map& m, const K k)
    {
            return m[k];
    }

    template <class K>
    static typename
    boost::remove_pointer<
            typename Map::value_type
    >::type::second_type&
    get(const Map& m, const K k)
    {
            return const_cast<Map&>(m)[k];
    }
};

template <class Map>
struct map_utils<Map*>
{
    template <class T>
    static typename
    boost::remove_pointer<
            typename Map::value_type
    >::type::second_type&
    get(Map* m, const T t)
    {
            return (*m)[t];
    }

    template <class T>
    static typename
    boost::remove_pointer<
            typename Map::value_type
    >::type::second_type&
    get(const Map* m, const T t)
    {
            return const_cast<Map*>(m)->operator[](t);
    }
};

I'm trying to use boost::mpl to do this, and this is what I've cooked up so far, but I get the same error using both versions of the code.

The error.

error: invalid initialization of reference of type ‘std::map<int, double>* const&’ from         expression of type     ‘boost::ptr_container_detail::reversible_ptr_container<boost::ptr_container_detail::map_config<std::map<int, double>, std::map<int, void*, std::less<int>, std::allocator<std::pair<const int, void*> > >, true>, boost::heap_clone_allocator>::Ty_’

The modified specialization of the struct to deal with l-values that are not pointers to maps.

template <class K>
    static typename
    boost::mpl::if_<
            boost::is_pointer<
                    typename boost::remove_pointer<
                            typename Map::value_type
                    >::type::second_type
            >,
            typename boost::remove_pointer<
                    typename boost::remove_const<
                            typename Map::value_type
                    >::type
            >::type::second_type,
            typename boost::remove_pointer<
                    typename Map::value_type
            >::type::second_type&
    >::type
    get(Map& m, const K k)
    {
            return m[k];
    }

C and F seems wrong, the mapped type isn't boost::ptr_map. Otherwise it sounds like you could just use full template specialization to decide whether it is an std::map or boost::ptr_map. Something like this:

template <class Map>
class Whatever;

template <class K, class V>
class Whatever<std::map<K, V> >
{
    public:
        typedef V& Type;
};

template <class K, class V>
class Whatever<std::map<K, V>* >
{
    public:
        typedef V& Type;
};

template <class K, class V>
class Whatever<boost::ptr_map<K, V> >
{
    public:
        typedef V* const Type;
};

template <class K, class V>
class Whatever<boost::ptr_map<K, V>* >
{
    public:
        typedef V* const Type;
};