Hive Generic UDF : Hive does not cast as expected, Caused by: java.lang.ClassCastException: java.util.ArrayList cannot be cast to java.util.Map

I am trying to create a simple generic udf for my hive queries.

Here is my hive table

CREATE TABLE `dum`(`val` map<string,array<string>>);
insert into dum select map('A',array('1','2','3'),'B',array('4','5','6'));

and here is how it looks

select * from dum;
{"A":["1","2","3"],"B":["4","5","6"]}

I am trying to create a simple UDF that can combine all the items in the values of the above map into a list. Here is what i want to see

select modudf(val) from dum;
["1","2","3","4","5","6"]

so i created

package some.package;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.udf.UDFType;
import java.util.ArrayList;

import java.util.List;
import java.util.Map;

@UDFType(deterministic = true)
public class CustomUDF extends UDF {

public List<String> evaluate(Map<String, List<String>> inMap) {

    List<String> res = new ArrayList<String>();
    for(Map.Entry<String, List<String>> ent : inMap.entrySet()){
        
        for(String item : ent.getValue())
            res.add(item);
    }
        return res;
    }
}

and this works perfectly as i try to invoke it

add jar /path/to/my/jar;
CREATE TEMPORARY FUNCTION modudf AS 'some.package.CustomUDF';
select modudf(val) from dum;

i get

["1","2","3","4","5","6"]

however, i want to create a generic udf and so i tried

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;

import org.apache.hadoop.hive.serde2.objectinspector.*;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;

import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class CustomUDF2 extends GenericUDF {
    private MapObjectInspector  inputMapOI                        = null;

    private Converter           inputMapKeyConverter              = null;

    private ListObjectInspector inputMapValueListOI               = null;
    private Converter           inputMapValueListElementConverter = null;

    @Override
    public String getDisplayString(String[] arguments) {
        return "Some message";
    }

    @Override
    public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
        if ((null == arguments) || (arguments.length != 1)) {
            throw new UDFArgumentLengthException("1 argument is expected.");
        }

        if (!(arguments[0] instanceof MapObjectInspector)) {
            throw new UDFArgumentException("The first parameter should be a map object.");
        }

        this.inputMapOI = (MapObjectInspector) arguments[0];

        ObjectInspector mapKeyOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
        this.inputMapKeyConverter = ObjectInspectorConverters.getConverter(this.inputMapOI.getMapKeyObjectInspector(), mapKeyOI);

        if (!(this.inputMapOI.getMapValueObjectInspector() instanceof ListObjectInspector)) {
            throw new UDFArgumentException("The map value type must be a list (aka array)");
        }
        this.inputMapValueListOI = (ListObjectInspector) this.inputMapOI.getMapValueObjectInspector();

        ObjectInspector inputListElementOI = this.inputMapValueListOI.getListElementObjectInspector();
        ObjectInspector outputListElementOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
        this.inputMapValueListElementConverter = ObjectInspectorConverters.getConverter(inputListElementOI, outputListElementOI);

        ObjectInspector outputMapKeyOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
        ObjectInspector outputMapValueListElementOI = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(PrimitiveObjectInspector.PrimitiveCategory.STRING);
        ObjectInspector outputMapValueListOI = ObjectInspectorFactory.getStandardListObjectInspector(outputMapValueListElementOI);
        return ObjectInspectorFactory.getStandardMapObjectInspector(outputMapKeyOI, outputMapValueListOI);
    }

    @Override
    public Object evaluate(DeferredObject[] arguments) throws HiveException {
        if ((null == arguments) || (arguments.length != 1)) {
            throw new UDFArgumentLengthException("1 argument is expected.");
        }

        Map<?, ?> map = inputMapOI.getMap(arguments[0].get());
        List<String> dataList = new ArrayList<String>();
        for (Object key : map.keySet()) {
            List<?> valueList = this.inputMapValueListOI.getList(map.get(key));
            if ((valueList == null) || (valueList.size() == 0)) {
                continue;
            }

            String strKey = (String) this.inputMapKeyConverter.convert(key);

            for (Object value : valueList) {
                String strValue = (String) this.inputMapValueListElementConverter.convert(value);
                dataList.add(strValue);
            }
        }

        return dataList;
    }
}

but this time when i invoke it i get an error

    add jar /path/to/my/jar;
    CREATE TEMPORARY FUNCTION modudf AS 'some.package.CustomUDF2';
    select modudf(val) from dum;

Caused by: org.apache.hadoop.hive.ql.metadata.HiveException: Hive Runtime Error while processing row {"val":{"A":["1","2","3"],"B":["4","5","6"]}}
    at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:562)
    at org.apache.hadoop.hive.ql.exec.mr.ExecMapper.map(ExecMapper.java:148)
    ... 8 more
Caused by: java.lang.ClassCastException: java.util.ArrayList cannot be cast to java.util.Map
    at org.apache.hadoop.hive.serde2.objectinspector.StandardMapObjectInspector.getMap(StandardMapObjectInspector.java:85)
    at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serialize(LazySimpleSerDe.java:321)
    at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.serializeField(LazySimpleSerDe.java:247)
    at org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.doSerialize(LazySimpleSerDe.java:231)
    at org.apache.hadoop.hive.serde2.AbstractEncodingAwareSerDe.serialize(AbstractEncodingAwareSerDe.java:55)
    at org.apache.hadoop.hive.ql.exec.FileSinkOperator.process(FileSinkOperator.java:725)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
    at org.apache.hadoop.hive.ql.exec.SelectOperator.process(SelectOperator.java:95)
    at org.apache.hadoop.hive.ql.exec.Operator.forward(Operator.java:897)
    at org.apache.hadoop.hive.ql.exec.TableScanOperator.process(TableScanOperator.java:130)
    at org.apache.hadoop.hive.ql.exec.MapOperator$MapOpCtx.forward(MapOperator.java:148)
    at org.apache.hadoop.hive.ql.exec.MapOperator.process(MapOperator.java:547)
    ... 9 more

As far as I can tell, i am never trying to convert ArrayList to Map.

What am i doing wrong ?

Solution

What you need to return in initialize is the ObjectInspector corresponding to the returning type of the UDF (ListObjectInspector in your case).

When you write

public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
    ...
    return ObjectInspectorFactory.getStandardMapObjectInspector(outputMapKeyOI, outputMapValueListOI);
}

Hive expects to find Map as the output. So when the code executed after evaluate finds a List, it throws an exception.