scalaakkaapache-zeppelin

an openapi client jar, generated for scala-akka by openapi-generator-maven-plugin, causes canBuildFromIterableViewMapLike() not found, under Zeppelin


I'm generating a Scala client jar with org.openapitools:openapi-generator-maven-plugin so that my Zeppelin will be able to execute this sample of code,

openapi-generator-maven-plugin offers few generators for Scala, and recommends to choose scala-akka.

It requires to produce a complicated piece of client code:

%spark
import java.util.Map;
import fr.ecoemploi.application.etude.swagger.invoker._
import fr.ecoemploi.application.etude.swagger.model.Commune;
import fr.ecoemploi.application.etude.swagger.api.CogControllerApi;
import fr.ecoemploi.application.etude.swagger.invoker.CollectionFormats._
import fr.ecoemploi.application.etude.swagger.invoker.ApiKeyLocations._

import akka.actor.ActorSystem
import akka.stream.ActorMaterializer

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.Future
import scala.util.{Failure, Success}

implicit val system = ActorSystem("system")
implicit val executionContext = system.dispatcher
implicit val materializer: ActorMaterializer = ActorMaterializer()

val apiInvoker = ApiInvoker()
val cogService = CogControllerApi("http://localhost:9090")
val request = cogService.obtenirCommunes(2022);

val response = apiInvoker.execute(request)

response.onComplete {
    case Success(ApiResponse(code, content, headers)) =>
        System.out.println(s"Status code: $code}")
        System.out.println(s"Response headers: ${headers.mkString(", ")}")
        System.out.println(s"Response body: $content")
        
    case Failure(error @ ApiError(code, message, responseContent, cause, headers)) =>
        System.err.println("Exception when calling CogControllerApi#obtenirCommunes")
        System.err.println(s"Status code: $code}")
        System.err.println(s"Reason: $responseContent")
        System.err.println(s"Response headers: ${headers.mkString(", ")}")
        error.printStackTrace();

    case Failure(exception) => 
        System.err.println("Exception when calling CogControllerApi#obtenirCommunes")
        exception.printStackTrace();
}

With eventually a run failure under Zeppelin:

java.lang.NoSuchMethodError: 'scala.collection.generic.CanBuildFrom scala.collection.compat.package$.canBuildFromIterableViewMapLike()'
  at fr.ecoemploi.application.etude.swagger.invoker.ApiInvoker.makeUri(ApiInvoker.scala:213)
  at fr.ecoemploi.application.etude.swagger.invoker.ApiInvoker.execute(ApiInvoker.scala:226)
  ... 44 elided

How I've managed things:

Its generated openapi.jar has this content:

jar tf openapi-client-1.0.0.jar 
META-INF/
META-INF/MANIFEST.MF
fr/
fr/ecoemploi/
fr/ecoemploi/application/
fr/ecoemploi/application/etude/
fr/ecoemploi/application/etude/swagger/
fr/ecoemploi/application/etude/swagger/model/
fr/ecoemploi/application/etude/swagger/api/
fr/ecoemploi/application/etude/swagger/invoker/
fr/ecoemploi/application/etude/swagger/model/Commune$.class
fr/ecoemploi/application/etude/swagger/model/Commune.class
fr/ecoemploi/application/etude/swagger/api/CogControllerApi$.class
fr/ecoemploi/application/etude/swagger/api/ComptesCollectivitesControllerApi.class
fr/ecoemploi/application/etude/swagger/api/ComptesCollectivitesControllerApi$.class
fr/ecoemploi/application/etude/swagger/api/EnumsSerializers$.class
fr/ecoemploi/application/etude/swagger/api/EnumsSerializers.class
fr/ecoemploi/application/etude/swagger/api/CovidControllerApi.class
fr/ecoemploi/application/etude/swagger/api/EquipementControllerApi$.class
fr/ecoemploi/application/etude/swagger/api/CovidControllerApi$.class
fr/ecoemploi/application/etude/swagger/api/EnumsSerializers$EnumNameSerializer.class
fr/ecoemploi/application/etude/swagger/api/IntercoControllerApi$.class
fr/ecoemploi/application/etude/swagger/api/ActivitesControllerApi.class
fr/ecoemploi/application/etude/swagger/api/EnumsSerializers$EnumNameSerializer$$anonfun$deserialize$1.class
fr/ecoemploi/application/etude/swagger/api/ActivitesControllerApi$.class
fr/ecoemploi/application/etude/swagger/api/AssociationsControllerApi$.class
fr/ecoemploi/application/etude/swagger/api/IntercoControllerApi.class
fr/ecoemploi/application/etude/swagger/api/EnumsSerializers$EnumNameSerializer$$anonfun$serialize$1.class
fr/ecoemploi/application/etude/swagger/api/EquipementControllerApi.class
fr/ecoemploi/application/etude/swagger/api/CogControllerApi.class
fr/ecoemploi/application/etude/swagger/api/AssociationsControllerApi.class
fr/ecoemploi/application/etude/swagger/invoker/ApiRequest$.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormat.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$LocalDateSerializer$.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$DateTimeSerializer$$anonfun$$lessinit$greater$1.class
fr/ecoemploi/application/etude/swagger/invoker/NumericValue.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyCredentials.class
fr/ecoemploi/application/etude/swagger/invoker/CustomContentTypes.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormats.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyValue.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$DateTimeSerializer$$anonfun$$lessinit$greater$1$$anonfun$apply$1.class
fr/ecoemploi/application/etude/swagger/invoker/ApiError$$anonfun$$lessinit$greater$1.class
fr/ecoemploi/application/etude/swagger/invoker/Credentials.class
fr/ecoemploi/application/etude/swagger/invoker/ParametersMap$.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormats$CSV$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyCredentials$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiModel.class
fr/ecoemploi/application/etude/swagger/invoker/ApiMethods.class
fr/ecoemploi/application/etude/swagger/invoker/ParametersMap.class
fr/ecoemploi/application/etude/swagger/invoker/BasicCredentials.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$DateTimeSerializer$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiMethod.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyLocations$HEADER$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyValue$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiInvoker$$anonfun$unmarshallApiResponse$4.class
fr/ecoemploi/application/etude/swagger/invoker/ApiSettings.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyLocations$QUERY$.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormats$SSV$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiInvoker$.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$LocalDateSerializer$$anonfun$$lessinit$greater$2.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$DateTimeSerializer$$anonfun$$lessinit$greater$1$$anonfun$apply$2.class
fr/ecoemploi/application/etude/swagger/invoker/ApiResponse.class
fr/ecoemploi/application/etude/swagger/invoker/ApiRequest.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyLocation.class
fr/ecoemploi/application/etude/swagger/invoker/ApiInvoker$ApiRequestImprovements.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormats$PIPES$.class
fr/ecoemploi/application/etude/swagger/invoker/UnitJSONSupport.class
fr/ecoemploi/application/etude/swagger/invoker/BearerToken$.class
fr/ecoemploi/application/etude/swagger/invoker/BasicCredentials$.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$.class
fr/ecoemploi/application/etude/swagger/invoker/NumericValue$.class
fr/ecoemploi/application/etude/swagger/invoker/MergedArrayFormat.class
fr/ecoemploi/application/etude/swagger/invoker/ApiResponse$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyLocations$COOKIE$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiMethod$.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$LocalDateSerializer$$anonfun$$lessinit$greater$2$$anonfun$apply$4.class
fr/ecoemploi/application/etude/swagger/invoker/BearerToken.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyLocations.class
fr/ecoemploi/application/etude/swagger/invoker/ResponseState$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiInvoker.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormats$TSV$.class
fr/ecoemploi/application/etude/swagger/invoker/ArrayValues.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormats$.class
fr/ecoemploi/application/etude/swagger/invoker/ResponseState$Error$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiError$.class
fr/ecoemploi/application/etude/swagger/invoker/ParametersMap$ParametersMapImprovements.class
fr/ecoemploi/application/etude/swagger/invoker/ApiMethods$.class
fr/ecoemploi/application/etude/swagger/invoker/CollectionFormats$MULTI$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiInvoker$ApiMethodExtensions.class
fr/ecoemploi/application/etude/swagger/invoker/ApiKeyLocations$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiReturnWithHeaders.class
fr/ecoemploi/application/etude/swagger/invoker/ApiSettings$.class
fr/ecoemploi/application/etude/swagger/invoker/ResponseState.class
fr/ecoemploi/application/etude/swagger/invoker/ApiError.class
fr/ecoemploi/application/etude/swagger/invoker/ResponseState$Success$.class
fr/ecoemploi/application/etude/swagger/invoker/ApiError$$anonfun$$lessinit$greater$2.class
fr/ecoemploi/application/etude/swagger/invoker/ArrayValues$.class
fr/ecoemploi/application/etude/swagger/invoker/Serializers$LocalDateSerializer$$anonfun$$lessinit$greater$2$$anonfun$apply$3.class
reference.conf
META-INF/maven/
META-INF/maven/org.openapitools/
META-INF/maven/org.openapitools/openapi-client/
META-INF/maven/org.openapitools/openapi-client/pom.xml
META-INF/maven/org.openapitools/openapi-client/pom.properties

The lib folder accompanying the generated sources when they are compiled, has this content:

akka-actor_2.12-2.6.12.jar        json4s-core_2.12-3.6.7.jar               scalatest-core_2.12-3.2.3.jar
akka-http_2.12-10.2.3.jar         json4s-ext_2.12-3.6.7.jar                scalatest-diagrams_2.12-3.2.3.jar
akka-http-core_2.12-10.2.3.jar    json4s-jackson_2.12-3.6.7.jar            scalatest-featurespec_2.12-3.2.3.jar
akka-http-json4s_2.12-1.27.0.jar  json4s-scalap_2.12-3.6.7.jar             scalatest-flatspec_2.12-3.2.3.jar
akka-parsing_2.12-10.2.3.jar      junit-4-13_2.12-3.2.3.0.jar              scalatest-freespec_2.12-3.2.3.jar
akka-protobuf-v3_2.12-2.6.12.jar  junit-4.13.jar                           scalatest-funspec_2.12-3.2.3.jar
akka-stream_2.12-2.6.12.jar       paranamer-2.8.jar                        scalatest-funsuite_2.12-3.2.3.jar
config-1.4.1.jar                  reactive-streams-1.0.3.jar               scalatest-matchers-core_2.12-3.2.3.jar
hamcrest-core-1.3.jar             scala-collection-compat_2.12-2.4.1.jar   scalatest-mustmatchers_2.12-3.2.3.jar
hpack-1.0.2.jar                   scalactic_2.12-3.2.3.jar                 scalatest-propspec_2.12-3.2.3.jar
jackson-annotations-2.9.0.jar     scala-java8-compat_2.12-0.8.0.jar        scalatest-refspec_2.12-3.2.3.jar
jackson-core-2.9.8.jar            scala-library-2.12.13.jar                scalatest-shouldmatchers_2.12-3.2.3.jar
jackson-databind-2.9.8.jar        scala-parser-combinators_2.12-1.1.2.jar  scalatest-wordspec_2.12-3.2.3.jar
joda-convert-2.2.0.jar            scala-reflect-2.12.12.jar                scala-xml_2.12-1.2.0.jar
joda-time-2.10.1.jar              scalatest_2.12-3.2.3.jar                 ssl-config-core_2.12-0.4.2.jar
json4s-ast_2.12-3.6.7.jar         scalatest-compatible-3.2.3.jar

It appears that on the spark.jars attribute of the Zeppelin Scala interpreter, I shall not only add
openapi-client-1.0.0.jar, but also some of these jars listed.

But not all of them, of course... I have joined those to spark.jars attribute of Zeppelin:

spark.jars  /home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/openapi-client-1.0.0.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/akka-actor_2.12-2.6.12.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/akka-http_2.12-10.2.3.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/akka-http-core_2.12-10.2.3.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/akka-http-json4s_2.12-1.27.0.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/akka-parsing_2.12-10.2.3.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/akka-protobuf-v3_2.12-2.6.12.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/akka-stream_2.12-2.6.12.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/config-1.4.1.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/hamcrest-core-1.3.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/hpack-1.0.2.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/paranamer-2.8.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/reactive-streams-1.0.3.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/jackson-annotations-2.9.0.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/jackson-core-2.9.8.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/jackson-databind-2.9.8.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/joda-convert-2.2.0.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/joda-time-2.10.1.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/json4s-ast_2.12-3.6.7.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/json4s-core_2.12-3.6.7.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/json4s-ext_2.12-3.6.7.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/json4s-jackson_2.12-3.6.7.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/json4s-scalap_2.12-3.6.7.jar,
/home/lebihan/dev/Java/comptes-france/metier-et-gestion/dev/GenerationOpenAPI/target/generated-sources/swagger/scala/target/lib/ssl-config-core_2.12-0.4.2.jar

May be I souldn't use scala-akka for client generator?

Or, am I on the wrong way for generation, and Apache Zeppelin requires another generator selection that scala-akka, for the openapi-client.jar to use with it?


Solution

  • Epilogue

    Scala is a hell of incompatible jars, even in the middle of its 2.12.x or 2.13.x versions.

    It happened that Spark (used by Apache Zeppelin) was using incompatible Scala jars from scala.collection.compat package, some coming through com.google.protobuf, did I red.

    So the workaround is to create a fat jar relocating these classes elsewhere, renaming their packages. Then, the client-openapi.jar created will be accepted.

      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-shade-plugin</artifactId>
        <executions>
          <execution>
            <phase>package</phase>
            <goals>
              <goal>shade</goal>
            </goals>
            <configuration>
              <transformers>
                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                  <mainClass></mainClass>
                </transformer>
                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                  <resource>reference.conf</resource>
                </transformer>
    
                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
              </transformers>
              <filters>
                <filter>
                  <artifact>*:*</artifact>
                  <excludes>
                    <exclude>META-INF/maven/**</exclude>
                    <exclude>META-INF/*.SF</exclude>
                    <exclude>META-INF/*.DSA</exclude>
                    <exclude>META-INF/*.RSA</exclude>
                  </excludes>
                </filter>
              </filters>
              <relocations>
                <relocation>
                  <pattern>com</pattern>
                  <shadedPattern>repackaged.com.google.common</shadedPattern>
    
                  <includes>
                    <include>com.google.common.**</include>
                  </includes>
                </relocation>
    
                <relocation>
                  <pattern>com.google.protobuf</pattern>
                  <shadedPattern>com.shaded.protobuf</shadedPattern>
    
                  <includes>
                    <include>com.google.protobuf.**</include>
                  </includes>
                </relocation>
    
                <relocation>
                  <pattern>scala.collection.compat</pattern>
                  <shadedPattern>scala.shaded.compat</shadedPattern>
    
                  <includes>
                    <include>scala.collection.compat.**</include>
                  </includes>
                </relocation>
    
                <relocation>
                  <pattern>shapeless</pattern>
                  <shadedPattern>shapelessshaded</shadedPattern>
    
                  <includes>
                    <include>shapeless.**</include>
                  </includes>
                </relocation>
              </relocations>
            </configuration>
          </execution>
        </executions>
      </plugin>