Trying to ingest shapefiles with over 600 columns in Geomesa I get the following error.
at org.locationtech.geomesa.index.geotools.GeoMesaFeatureWriter.writeFeature(GeoMesaFeatureWriter.scala:56)
at org.locationtech.geomesa.index.geotools.GeoMesaFeatureWriter.writeFeature$(GeoMesaFeatureWriter.scala:46)
at org.locationtech.geomesa.index.geotools.GeoMesaFeatureWriter$TableFeatureWriter.writeFeature(GeoMesaFeatureWriter.scala:151)
at org.locationtech.geomesa.index.geotools.GeoMesaFeatureWriter$GeoMesaAppendFeatureWriter.write(GeoMesaFeatureWriter.scala:239)
at org.locationtech.geomesa.index.geotools.GeoMesaFeatureWriter$GeoMesaAppendFeatureWriter.write$(GeoMesaFeatureWriter.scala:235)
at org.locationtech.geomesa.index.geotools.GeoMesaFeatureWriter$$anon$3.write(GeoMesaFeatureWriter.scala:111)
at org.locationtech.geomesa.utils.geotools.FeatureUtils$.write(FeatureUtils.scala:147)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$8(LocalConverterIngest.scala:181)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$8$adapted(LocalConverterIngest.scala:179)
at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:575)
at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:573)
at org.locationtech.geomesa.utils.collection.CloseableIterator$FlatMapCloseableIterator.foreach(CloseableIterator.scala:132)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$7(LocalConverterIngest.scala:179)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$7$adapted(LocalConverterIngest.scala:173)
at org.locationtech.geomesa.utils.io.CloseablePool$CommonsPoolPool.borrow(CloseablePool.scala:68)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$6(LocalConverterIngest.scala:173)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$6$adapted(LocalConverterIngest.scala:172)
at org.locationtech.geomesa.utils.io.package$WithClose$.apply(package.scala:64)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$4(LocalConverterIngest.scala:172)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$4$adapted(LocalConverterIngest.scala:168)
at scala.collection.IterableOnceOps.foreach(IterableOnce.scala:575)
at scala.collection.IterableOnceOps.foreach$(IterableOnce.scala:573)
at org.locationtech.geomesa.utils.collection.CloseableIterator$CloseableSingleIterator.foreach(CloseableIterator.scala:85)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$3(LocalConverterIngest.scala:168)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$3$adapted(LocalConverterIngest.scala:167)
at org.locationtech.geomesa.utils.io.package$WithClose$.apply(package.scala:64)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$2(LocalConverterIngest.scala:167)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.$anonfun$run$2$adapted(LocalConverterIngest.scala:166)
at org.locationtech.geomesa.utils.io.CloseablePool$CommonsPoolPool.borrow(CloseablePool.scala:68)
at org.locationtech.geomesa.tools.ingest.LocalConverterIngest$LocalIngestWorker.run(LocalConverterIngest.scala:166)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.ArrayIndexOutOfBoundsException: Index 2136 out of bounds for length 1024
at com.esotericsoftware.kryo.io.Output.writeByte(Output.java:226)
at jdk.internal.reflect.GeneratedMethodAccessor37.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.locationtech.geomesa.features.serialization.WkbSerialization.serializeWkb(WkbSerialization.scala:44)
at org.locationtech.geomesa.features.serialization.WkbSerialization.serializeWkb$(WkbSerialization.scala:42)
at org.locationtech.geomesa.features.kryo.serialization.KryoGeometrySerialization$.serializeWkb(KryoGeometrySerialization.scala:14)
at org.locationtech.geomesa.features.kryo.impl.KryoFeatureSerialization$KryoGeometryWkbWriter$.apply(KryoFeatureSerialization.scala:229)
at org.locationtech.geomesa.features.kryo.impl.KryoFeatureSerialization.writeFeature(KryoFeatureSerialization.scala:71)
at org.locationtech.geomesa.features.kryo.impl.KryoFeatureSerialization.serialize(KryoFeatureSerialization.scala:43)
at org.locationtech.geomesa.features.kryo.impl.KryoFeatureSerialization.serialize$(KryoFeatureSerialization.scala:41)
at org.locationtech.geomesa.features.kryo.KryoFeatureSerializer$MutableActiveSerializer.serialize(KryoFeatureSerializer.scala:75)
at org.locationtech.geomesa.index.api.WritableFeature$FeatureLevelWritableFeature.$anonfun$values$2(WritableFeature.scala:153)
at org.locationtech.geomesa.index.api.package$KeyValue.value$lzycompute(package.scala:183)
at org.locationtech.geomesa.index.api.package$KeyValue.value(package.scala:183)
at org.locationtech.geomesa.accumulo.data.AccumuloIndexAdapter$AccumuloIndexWriter.$anonfun$write$1(AccumuloIndexAdapter.scala:397)
at org.locationtech.geomesa.accumulo.data.AccumuloIndexAdapter$AccumuloIndexWriter.$anonfun$write$1$adapted(AccumuloIndexAdapter.scala:396)
at scala.collection.immutable.Vector.foreach(Vector.scala:1895)
at org.locationtech.geomesa.accumulo.data.AccumuloIndexAdapter$AccumuloIndexWriter.write(AccumuloIndexAdapter.scala:396)
at org.locationtech.geomesa.index.api.IndexAdapter$BaseIndexWriter.write(IndexAdapter.scala:153)
at org.locationtech.geomesa.index.geotools.GeoMesaFeatureWriter.writeFeature(GeoMesaFeatureWriter.scala:50)
... 34 more
The error occurs on both older and newer versions of Geomesa. I have tried installing the latest versions of some key libraries, such as GeoTools and Kryo, but the problem persists.
Could it make sense to change the way the data is serialized? If so, can it be done without recompiling the code?
The issue is a bug in GeoMesa affecting schemas with > 479 attributes. It has been fixed in main, and will be available in the next release (4.1.0). In the meantime, you can build from source if you need the fix immediately.