javaspringspring-bootapache-spark

java.lang.NoClassDefFoundError: javax/servlet/http/HttpServletRequest with Apache Spark in SpringBoot with ApplicationRunner


I am trying to run Apache Spark 3.3.2 with Spring Boot 2.6.6 with ApplicationRunner Java 17 Mac OS; I have set this in the application.properties: spring.main.web-application-type=none

Spring Application

@SpringBootApplication
public class MyApplication implements ApplicationRunner {

private final MyService myService;

public MyApplication(MyService myService) {
    this.myService = myService;
}

public static void main(String[] args) {
    SpringApplication.run(MyApplication.class, args);
}

@Override
public void run(ApplicationArguments args) throws Exception {
    myService.process(args);
}
}

Apache Spark Session in MyService

SparkSession spark = SparkSession.builder()
            .master("local[*]")
            .config("spark.ui.enabled", "false")
            .getOrCreate();

Maven pom.xml

       <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.13</artifactId>
            <version>3.3.2</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.13</artifactId>
            <version>3.3.2</version>
        </dependency>

I get this error when I try to start the Spark Session:

java.lang.NoClassDefFoundError: javax/servlet/http/HttpServletRequest
    at org.apache.spark.metrics.sink.MetricsServlet.getHandlers(MetricsServlet.scala:51) ~[spark-core_2.13-3.3.2.jar:3.3.2]
    at org.apache.spark.metrics.MetricsSystem.$anonfun$getServletHandlers$2(MetricsSystem.scala:90) ~[spark-core_2.13-3.3.2.jar:3.3.2]
    at scala.Option.map(Option.scala:242) ~[scala-library-2.13.8.jar:?]
    at org.apache.spark.metrics.MetricsSystem.getServletHandlers(MetricsSystem.scala:90) ~[spark-core_2.13-3.3.2.jar:3.3.2]
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:660) ~[spark-core_2.13-3.3.2.jar:3.3.2]
    at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2714) ~[spark-core_2.13-3.3.2.jar:3.3.2]
    at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:953) ~[spark-sql_2.13-3.3.2.jar:3.3.2]
    at scala.Option.getOrElse(Option.scala:201) ~[scala-library-2.13.8.jar:?]
    at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:947) ~[spark-sql_2.13-3.3.2.jar:3.3.2]
    at com.boomi.intelligence.MyService.process(MyService.java:48) ~[classes/:?]
    at com.boomi.intelligence.MyApplication.run(MyApplication.java:31) ~[classes/:?]
    at org.springframework.boot.SpringApplication.callRunner(SpringApplication.java:760) ~[spring-boot-3.0.4.jar:3.0.4]
    at org.springframework.boot.SpringApplication.callRunners(SpringApplication.java:750) ~[spring-boot-3.0.4.jar:3.0.4]
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:317) ~[spring-boot-3.0.4.jar:3.0.4]
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:1304) ~[spring-boot-3.0.4.jar:3.0.4]
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:1293) ~[spring-boot-3.0.4.jar:3.0.4]
    at com.boomi.intelligence.MyApplication.main(MyApplication.java:24) ~[classes/:?]
Caused by: java.lang.ClassNotFoundException: javax.servlet.http.HttpServletRequest
    at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?]
    at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?]
    at java.lang.ClassLoader.loadClass(ClassLoader.java:520) ~[?:?]

Solution

  • Thank you for the responses. Although, jakarata.servlet-api already exists in Apache Spark. It still needed javax.servlet-api. Adding just javax.servlet-api did not resolve the issue. I was able to resolve this issue by adding these in the pom.xml

          <dependency>
             <groupId>javax.servlet</groupId>
             <artifactId>javax.servlet-api</artifactId>
             <version>3.1.0</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
            <groupId>org.antlr</groupId>
            <artifactId>antlr4-runtime</artifactId>
            <version>4.8</version>
        </dependency>
        <dependency>
           <artifactId>janino</artifactId>
           <groupId>org.codehaus.janino</groupId>
          <version>3.0.8</version>
        </dependency>
        <dependency>
           <artifactId>commons-compiler</artifactId>
           <groupId>org.codehaus.janino</groupId>
           <version>3.0.8</version>
        </dependency>