While processing a JSON message with AVRO schema, I am getting error "org.apache.avro.SchemaParseException: Undefined name: 'array'". Not able to identify where the exact problem is.
Source Schema is
{
"eventId" : "123456",
"documentType" : "DocCreation",
"metadata" : {
"subType" : "DepositCreation",
"sourceSystem" : "SystemA",
"sourceApplication" : "CustomerApplication"
},
"documentRef" : "documentRef",
"payload" : {
"addressCountry" : "US",
"sms1Group" : [ {
"email1" : "email1@email.com",
"index" : "1",
"sms1" : "222222222"
}, {
"email1" : "email2@email.com",
"index" : "2",
"sms1" : "444444444"
} ],
"countryList" : {
"country" : [ {
"country" : "USA",
"language" : "GB"
} ]
},
"name1List" : {
"name1" : [ {
"language" : "GB",
"name1" : "JonDoe Fullname TEST"
} ]
},
"shortNameList" : {
"shortName" : [ {
"language" : "GB",
"shortName" : "JonDoe short name updated AGAIAN"
}, {
"language" : "LT",
"shortName" : "JonDoe short name in LT"
} ]
},
"addressList" : {
"address" : [ {
"address" : "My Address Line one in USA",
"index" : "1",
"language" : "GB"
}, {
"address" : "My Address Line one in LT",
"index" : "1",
"language" : "LT"
}, {
"address" : "My Address Line two in USA",
"index" : "2",
"language" : "GB"
} ]
},
"postCodeList" : {
"postCode" : [ {
"postCode" : "POSTCODE12356",
"language" : "GB"
} ]
},
"mnemonic" : "MSELVAN5",
"name2List" : {
"name2" : [ {
"language" : "GB",
"name2" : "JonDoe Full Name TWO"
} ]
},
"streetList" : {
"street" : [ {
"street" : "My Street in US",
"language" : "GB"
}, {
"street" : "My Street in LT",
"language" : "LT"
} ]
},
"id" : "100137",
"sector" : "1001"
},
"companyCode" : "ABCD123"
}
Avro Schema is
{
"type": "record",
"name": "Test",
"namespace": "com.group.Test",
"doc": "Here you can give more details about the use of this schema",
"fields": [
{
"name": "eventId",
"type": "string"
},
{
"name": "documentType",
"type": "string"
},
{
"name": "metadata",
"type": [
"null",
{
"type": "map",
"values": "string",
"avro.java.string": "String"
}
],
"doc": "Metadata",
"default": null
},
{
"name": "documentRef",
"type": "string"
},
{
"name": "companyCode",
"type": "string"
},
{
"name": "payload",
"type": {
"type": "record",
"name": "payload",
"fields": [
{
"name": "addressCountry",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "sms1Group",
"type": [
"null",
"array"
],
"items": {
"name": "sms1Group",
"type": "record",
"fields": [
{
"name": "email1",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "index",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "sms1",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
},
{
"name": "countryList",
"type": [
"null",
{
"name": "countryList",
"type": "record",
"fields": [
{
"name": "country",
"type": [
"null",
"array"
],
"items": {
"name": "country",
"type": "record",
"fields": [
{
"name": "country",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "language",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
}
]
}
],
"default": null
},
{
"name": "name1List",
"type": [
"null",
{
"name": "name1List",
"type": "record",
"fields": [
{
"name": "name1",
"type": [
"null",
"array"
],
"items": {
"name": "name1",
"type": "record",
"fields": [
{
"name": "language",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "name1",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
}
]
}
],
"default": null
},
{
"name": "shortNameList",
"type": [
"null",
{
"name": "shortNameList",
"type": "record",
"fields": [
{
"name": "shortName",
"type": [
"null",
"array"
],
"items": {
"name": "shortName",
"type": "record",
"fields": [
{
"name": "language",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "shortName",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
}
]
}
],
"default": null
},
{
"name": "addressList",
"type": [
"null",
{
"name": "addressList",
"type": "record",
"fields": [
{
"name": "address",
"type": [
"null",
"array"
],
"items": {
"name": "address",
"type": "record",
"fields": [
{
"name": "address",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "index",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "language",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
}
]
}
],
"default": null
},
{
"name": "postCodeList",
"type": [
"null",
{
"name": "postCodeList",
"type": "record",
"fields": [
{
"name": "postCode",
"type": [
"null",
"array"
],
"items": {
"name": "postCode",
"type": "record",
"fields": [
{
"name": "postCode",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "language",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
}
]
}
],
"default": null
},
{
"name": "mnemonic",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "name2List",
"type": [
"null",
{
"name": "name2List",
"type": "record",
"fields": [
{
"name": "name2",
"type": [
"null",
"array"
],
"items": {
"name": "name2",
"type": "record",
"fields": [
{
"name": "language",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "name2",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
}
]
}
],
"default": null
},
{
"name": "streetList",
"type": [
"null",
{
"name": "streetList",
"type": "record",
"fields": [
{
"name": "street",
"type": [
"null",
"array"
],
"items": {
"name": "street",
"type": "record",
"fields": [
{
"name": "street",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "language",
"type": [
"null",
"string"
],
"default": null
}
]
},
"default": null
}
]
}
],
"default": null
},
{
"name": "id",
"type": [
"null",
"string"
],
"default": null
},
{
"name": "sector",
"type": [
"null",
"string"
],
"default": null
}
]
}
},
{
"name": "companyCode",
"type": [
"null",
"string"
],
"default": null
}
]
}
Error is
ms.stacktrace="java.lang.refle
ct.InvocationTargetException
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
at java.base/java.lang.reflect.Method.invoke(Unknown Source)
at com.mycompany.microservice.framework.core.function.BaseProcessor.processEvent(BaseProcessor.java:1845)
at com.mycompany.microservice.framework.core.function.BaseProcessor.processMessage(BaseProcessor.java:1884)
at com.mycompany.microservice.framework.core.function.BaseProcessor.invokeST(BaseProcessor.java:669)
at com.mycompany.microservice.framework.core.ingester.GenericCommandSTBinaryIngester.process(GenericCommandSTBinaryIngester.java:212)
at com.mycompany.microservice.framework.core.ingester.GenericCommandSTBinaryIngester.initiateProcess(GenericCommandSTBinaryIngester.java:257)
at com.mycompany.microservice.framework.core.ingester.BinaryIngester.process(BinaryIngester.java:129)
at com.mycompany.microservice.framework.core.ingester.BinaryIngester.invoke(BinaryIngester.java:69)
at com.mycompany.microservice.framework.core.ingester.BinaryIngester.invoke(BinaryIngester.java:33)
at com.mycompany.microservice.framework.core.ingester.GenericEventIngestionProcessor.ingesterInvoker(GenericEventIngestionProcessor.java:81)
at com.mycompany.microservice.framework.core.ingester.GenericEventIngestionProcessor.ingest(GenericEventIngestionProcessor.java:148)
at com.mycompany.microservice.framework.core.ingester.GenericEventIngestionProcessor.process(GenericEventIngestionProcessor.java:231)
at com.mycompany.des.streamprocessor.topology.StreamTopology$DataProcessor.processData(StreamTopology.java:231)
at com.mycompany.des.streamprocessor.topology.StreamTopology$SourceProcessor.processData(StreamTopology.java:198)
at com.mycompany.des.streamprocessor.stream.state.EventStreamRunningState.runLoop(EventStreamRunningState.java:39)
at com.mycompany.des.streamprocessor.stream.EventStream.start(EventStream.java:112)
at com.mycompany.microservice.framework.ingester.thread.KafkaIngesterThread.run(KafkaIngesterThread.java:49)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source)
at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at java.base/java.lang.Thread.run(Unknown Source)
Caused by: org.apache.avro.SchemaParseException: Undefined name: 'array'
at org.apache.avro.Schema.parse(Schema.java:1697)
at org.apache.avro.Schema.parse(Schema.java:1823)
at org.apache.avro.Schema.parse(Schema.java:1736)
at org.apache.avro.Schema.parse(Schema.java:1736)
at org.apache.avro.Schema$Parser.parse(Schema.java:1471)
at org.apache.avro.Schema$Parser.parse(Schema.java:1458)
at com.mycompany.ecosystem.events.avro.AvroUtil.toSchema(AvroUtil.java:30)
at com.mycompany.microservice.dmdatadistributor.transforms.TransformsEntitySource.addSchema(TransformsEntitySource.java:77)
at com.mycompany.microservice.dmdatadistributor.transforms.TransformsEntitySource.initializeForDataType(TransformsEntitySource.java:57)
at com.mycompany.microservice.dmdatadistributor.transforms.TransformsEntitySource.get(TransformsEntitySource.java:34)
at com.mycompany.microservice.dmdatadistributor.handler.DataEventsHandler.handle(DataEventsHandler.java:47)
at com.mycompany.microservice.dmdatadistributor.ingester.EventHandlerFacade.processEvent(EventHandlerFacade.java:39)
In the Avro schema, I see fields defined like this:
{
"name": ...,
"type": [
"null",
"array"
],
"items": ...
}
In other words: your field has a custom property items (which is kept, but otherwise ignored by Avro), and your type is a union of null and array, which are both strings / names. But as there is no type named array, parsing fails.
To fix, construct your fields/types like this:
{
"name": ...,
"type": [
"null",
{
"type": "array",
"items": ...
}
]
}