hadoopmrunitmultipleoutputs

java.lang.ClassCastException: org.apache.hadoop.io.serializer.WritableSerialization cannot be cast to org.apache.hadoop.io.serializer.Serialization


I am testing MultipleOutputs using MRUnit. The test cases are getting failed with the following message.

java.lang.ClassCastException: org.apache.hadoop.io.serializer.WritableSerialization cannot be cast to org.apache.hadoop.io.serializer.Serialization

I am using the following annotations as per the examples I saw.

@RunWith(PowerMockRunner.class)
@PrepareForTest({MultipleOutputs.class, LoadMapper.class, LoadReducer.class})

For reference I am adding the code that I am using for testing.

@RunWith(PowerMockRunner.class)
@PrepareForTest({MultipleOutputs.class, LoadMapper.class, LoadReducer.class})
@PowerMockIgnore({
    "org.apache.log4j.*",
    "javax.xml.*",
    "org.xml.sax.*", 
    "org.apache.xerces.*",
    "org.w3c.dom.*"
    })
public class LoadTest extends LoadReducer {

    MapDriver<LongWritable, Text, NullWritable, Text> mapDriver;
    Configuration conf = new Configuration();

    @SuppressWarnings("deprecation")
    @Before
    public void setUp() {
        conf.set(LoadConstants.START_TAG_KEY, LoadConstants.START_TAG_KEY);
        conf.set(LoadConstants.END_TAG_KEY, LoadConstants.END_TAG_KEY);
        LoadMapper loadMapper = new LoadMapper();
        mapDriver = MapDriver.newMapDriver(loadMapper);
        mapDriver.setConfiguration(conf);
    }

    @Test
    public void testFailure() throws IOException {
        String inputRecord = "";
        String errorMsg = "";

        mapDriver.withInput(new LongWritable(0), new Text(inputRecord))
            .withMultiOutput(LoadConstants.BAD, NullWritable.get(), new Text(inputRecord + errorMsg))
            .runTest();

    }
}

The stack trace for the error is as below:

java.lang.LinkageError: loader constraint violation: when resolving method "org.apache.hadoop.io.serializer.SerializationFactory.<init>(Lorg/apache/hadoop/conf/Configuration;)V" the class loader (instance of org/powermock/core/classloader/MockClassLoader) of the current class, org/apache/hadoop/mrunit/internal/io/Serialization, and the class loader (instance of sun/misc/Launcher$AppClassLoader) for the method's defining class, org/apache/hadoop/io/serializer/SerializationFactory, have different Class objects for the type org/apache/hadoop/conf/Configuration used in the signature
    at org.apache.hadoop.mrunit.internal.io.Serialization.<init>(Serialization.java:39)
    at org.apache.hadoop.mrunit.TestDriver.getSerialization(TestDriver.java:530)
    at org.apache.hadoop.mrunit.TestDriver.copy(TestDriver.java:675)
    at org.apache.hadoop.mrunit.TestDriver.copyPair(TestDriver.java:679)
    at org.apache.hadoop.mrunit.MapDriverBase.addInput(MapDriverBase.java:120)
    at org.apache.hadoop.mrunit.MapDriverBase.withInput(MapDriverBase.java:210)
    at x.x.x.LoadTest.testFailure(LoadTest.java:20)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.junit.internal.runners.TestMethod.invoke(TestMethod.java:66)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit44RunnerDelegateImpl$PowerMockJUnit44MethodRunner.runTestMethod(PowerMockJUnit44RunnerDelegateImpl.java:310)
    at org.junit.internal.runners.MethodRoadie$2.run(MethodRoadie.java:86)
    at org.junit.internal.runners.MethodRoadie.runBeforesThenTestThenAfters(MethodRoadie.java:94)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit44RunnerDelegateImpl$PowerMockJUnit44MethodRunner.executeTest(PowerMockJUnit44RunnerDelegateImpl.java:294)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit47RunnerDelegateImpl$PowerMockJUnit47MethodRunner.executeTestInSuper(PowerMockJUnit47RunnerDelegateImpl.java:127)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit47RunnerDelegateImpl$PowerMockJUnit47MethodRunner.executeTest(PowerMockJUnit47RunnerDelegateImpl.java:82)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit44RunnerDelegateImpl$PowerMockJUnit44MethodRunner.runBeforesThenTestThenAfters(PowerMockJUnit44RunnerDelegateImpl.java:282)
    at org.junit.internal.runners.MethodRoadie.runTest(MethodRoadie.java:84)
    at org.junit.internal.runners.MethodRoadie.run(MethodRoadie.java:49)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit44RunnerDelegateImpl.invokeTestMethod(PowerMockJUnit44RunnerDelegateImpl.java:207)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit44RunnerDelegateImpl.runMethods(PowerMockJUnit44RunnerDelegateImpl.java:146)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit44RunnerDelegateImpl$1.run(PowerMockJUnit44RunnerDelegateImpl.java:120)
    at org.junit.internal.runners.ClassRoadie.runUnprotected(ClassRoadie.java:34)
    at org.junit.internal.runners.ClassRoadie.runProtected(ClassRoadie.java:44)
    at org.powermock.modules.junit4.internal.impl.PowerMockJUnit44RunnerDelegateImpl.run(PowerMockJUnit44RunnerDelegateImpl.java:118)
    at org.powermock.modules.junit4.common.internal.impl.JUnit4TestSuiteChunkerImpl.run(JUnit4TestSuiteChunkerImpl.java:101)
    at org.powermock.modules.junit4.common.internal.impl.AbstractCommonPowerMockRunner.run(AbstractCommonPowerMockRunner.java:53)
    at org.powermock.modules.junit4.PowerMockRunner.run(PowerMockRunner.java:53)
    at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
    at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)

POM dependency added for MRUnit:

<dependency>
    <groupId>org.apache.mrunit</groupId>
    <artifactId>mrunit</artifactId>
    <version>1.1.0</version>
    <classifier>hadoop2</classifier>
</dependency>

Hadoop version: 2.4.1 Environment : Windows 7 - 64 bit

If anyone have faced this issue please help.


Solution

  • The issue got fixed when the mapDriver.setConfiguration was removed from the test class.

     @SuppressWarnings("deprecation")
        @Before
        public void setUp() {
            conf.set(LoadConstants.START_TAG_KEY, LoadConstants.START_TAG_KEY);
            conf.set(LoadConstants.END_TAG_KEY, LoadConstants.END_TAG_KEY);
            LoadMapper loadMapper = new LoadMapper();
            mapDriver = MapDriver.newMapDriver(loadMapper);
            mapDriver.setConfiguration(conf); // Removed this line
        }
    

    I am not sure of the reason.