I am trying to configure a mirror maker between two kafkas and the data mirroring works fine for topics without compression. When mirroring a topic with a snappy compression I get the following exception:
java.io.FileNotFoundException: /tmp/snappy-1.1.10-00c148a7-468e-478a-855c-cabd60c7f21d-libsnappyjava.so (Read-only file system)
at java.base/java.io.FileOutputStream.open0(Native Method)
at java.base/java.io.FileOutputStream.open(FileOutputStream.java:298)
at java.base/java.io.FileOutputStream.<init>(FileOutputStream.java:237)
at java.base/java.io.FileOutputStream.<init>(FileOutputStream.java:187)
at org.xerial.snappy.SnappyLoader.extractLibraryFile(SnappyLoader.java:242)
at org.xerial.snappy.SnappyLoader.findNativeLibrary(SnappyLoader.java:358)
at org.xerial.snappy.SnappyLoader.loadNativeLibrary(SnappyLoader.java:179)
at org.xerial.snappy.SnappyLoader.loadSnappyApi(SnappyLoader.java:157)
at org.xerial.snappy.Snappy.init(Snappy.java:70)
at org.xerial.snappy.Snappy.<clinit>(Snappy.java:47)
at org.xerial.snappy.SnappyInputStream.hasNextChunk(SnappyInputStream.java:446)
at org.xerial.snappy.SnappyInputStream.read(SnappyInputStream.java:477)
at java.base/java.io.DataInputStream.readByte(DataInputStream.java:270)
at org.apache.kafka.common.utils.ByteUtils.readUnsignedVarint(ByteUtils.java:170)
at org.apache.kafka.common.utils.ByteUtils.readVarint(ByteUtils.java:205)
at org.apache.kafka.common.record.DefaultRecord.readFrom(DefaultRecord.java:296)
at org.apache.kafka.common.record.DefaultRecordBatch$2.doReadRecord(DefaultRecordBatch.java:278)
at org.apache.kafka.common.record.DefaultRecordBatch$StreamRecordIterator.readNext(DefaultRecordBatch.java:617)
at org.apache.kafka.common.record.DefaultRecordBatch$RecordIterator.next(DefaultRecordBatch.java:582)
at org.apache.kafka.common.record.DefaultRecordBatch$RecordIterator.next(DefaultRecordBatch.java:551)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.nextFetchedRecord(Fetcher.java:1556)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.fetchRecords(Fetcher.java:1591)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.access$1700(Fetcher.java:1432)
at org.apache.kafka.clients.consumer.internals.Fetcher.fetchRecords(Fetcher.java:684)
at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:635)
at org.apache.kafka.clients.consumer.KafkaConsumer.pollForFetches(KafkaConsumer.java:1303)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1237)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1210)
at kafka.tools.MirrorMaker$ConsumerWrapper.receive(MirrorMaker.scala:325)
at kafka.tools.MirrorMaker$MirrorMakerThread.run(MirrorMaker.scala:214)
[2023-11-09 20:30:08,767] ERROR [mirrormaker-thread-1] Mirror maker thread failure due to (kafka.tools.MirrorMaker$MirrorMakerThread)
java.lang.UnsatisfiedLinkError: no snappyjava in java.library.path: [/usr/java/packages/lib, /usr/lib64, /lib64, /lib, /usr/lib]
at java.base/java.lang.ClassLoader.loadLibrary(ClassLoader.java:2678)
at java.base/java.lang.Runtime.loadLibrary0(Runtime.java:830)
at java.base/java.lang.System.loadLibrary(System.java:1886)
at org.xerial.snappy.SnappyLoader.loadNativeLibrary(SnappyLoader.java:185)
at org.xerial.snappy.SnappyLoader.loadSnappyApi(SnappyLoader.java:157)
at org.xerial.snappy.Snappy.init(Snappy.java:70)
at org.xerial.snappy.Snappy.<clinit>(Snappy.java:47)
at org.xerial.snappy.SnappyInputStream.hasNextChunk(SnappyInputStream.java:446)
at org.xerial.snappy.SnappyInputStream.read(SnappyInputStream.java:477)
at java.base/java.io.DataInputStream.readByte(DataInputStream.java:270)
at org.apache.kafka.common.utils.ByteUtils.readUnsignedVarint(ByteUtils.java:170)
at org.apache.kafka.common.utils.ByteUtils.readVarint(ByteUtils.java:205)
at org.apache.kafka.common.record.DefaultRecord.readFrom(DefaultRecord.java:296)
at org.apache.kafka.common.record.DefaultRecordBatch$2.doReadRecord(DefaultRecordBatch.java:278)
at org.apache.kafka.common.record.DefaultRecordBatch$StreamRecordIterator.readNext(DefaultRecordBatch.java:617)
at org.apache.kafka.common.record.DefaultRecordBatch$RecordIterator.next(DefaultRecordBatch.java:582)
at org.apache.kafka.common.record.DefaultRecordBatch$RecordIterator.next(DefaultRecordBatch.java:551)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.nextFetchedRecord(Fetcher.java:1556)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.fetchRecords(Fetcher.java:1591)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.access$1700(Fetcher.java:1432)
at org.apache.kafka.clients.consumer.internals.Fetcher.fetchRecords(Fetcher.java:684)
at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:635)
at org.apache.kafka.clients.consumer.KafkaConsumer.pollForFetches(KafkaConsumer.java:1303)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1237)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1210)
at kafka.tools.MirrorMaker$ConsumerWrapper.receive(MirrorMaker.scala:325)
at kafka.tools.MirrorMaker$MirrorMakerThread.run(MirrorMaker.scala:214)
[2023-11-09 20:30:08,767] ERROR [mirrormaker-thread-2] Mirror maker thread failure due to (kafka.tools.MirrorMaker$MirrorMakerThread)
java.lang.NoClassDefFoundError: Could not initialize class org.xerial.snappy.Snappy
at org.xerial.snappy.SnappyInputStream.hasNextChunk(SnappyInputStream.java:446)
at org.xerial.snappy.SnappyInputStream.read(SnappyInputStream.java:477)
at java.base/java.io.DataInputStream.readByte(DataInputStream.java:270)
at org.apache.kafka.common.utils.ByteUtils.readUnsignedVarint(ByteUtils.java:170)
at org.apache.kafka.common.utils.ByteUtils.readVarint(ByteUtils.java:205)
at org.apache.kafka.common.record.DefaultRecord.readFrom(DefaultRecord.java:296)
at org.apache.kafka.common.record.DefaultRecordBatch$2.doReadRecord(DefaultRecordBatch.java:278)
at org.apache.kafka.common.record.DefaultRecordBatch$StreamRecordIterator.readNext(DefaultRecordBatch.java:617)
at org.apache.kafka.common.record.DefaultRecordBatch$RecordIterator.next(DefaultRecordBatch.java:582)
at org.apache.kafka.common.record.DefaultRecordBatch$RecordIterator.next(DefaultRecordBatch.java:551)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.nextFetchedRecord(Fetcher.java:1556)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.fetchRecords(Fetcher.java:1591)
at org.apache.kafka.clients.consumer.internals.Fetcher$CompletedFetch.access$1700(Fetcher.java:1432)
at org.apache.kafka.clients.consumer.internals.Fetcher.fetchRecords(Fetcher.java:684)
at org.apache.kafka.clients.consumer.internals.Fetcher.fetchedRecords(Fetcher.java:635)
at org.apache.kafka.clients.consumer.KafkaConsumer.pollForFetches(KafkaConsumer.java:1303)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1237)
at org.apache.kafka.clients.consumer.KafkaConsumer.poll(KafkaConsumer.java:1210)
at kafka.tools.MirrorMaker$ConsumerWrapper.receive(MirrorMaker.scala:325)
at kafka.tools.MirrorMaker$MirrorMakerThread.run(MirrorMaker.scala:214)
Docker image: confluentinc/cp-kafka:6.0.15 AND confluentinc/cp-kafka:5.4.1
Tried copying the jar file: COPY snappy-java-1.1.10.5.jar /lib/snappy-java-1.1.10.5.jar did not help.
Following the discussion below did not help as well
It turns out that the mirror maker does not come with Snappy Lib included. What I did was to copy libsnappyjava.so into /usr/lib/ of the docker image and it worked. I've taken the .so file from the github repo here
FROM confluentinc/cp-kafka:6.0.15
COPY libsnappyjava.so /usr/lib/libsnappyjava.so
USER root
RUN chmod 755 /usr/lib/libsnappyjava.so
USER appuser