I am attempting to write a basic loader for a MongoDb collection which will read prepared documents from a file and insert them into a collection The class I am using is currently written as below:
package com.example;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
import org.bson.Document;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.mongodb.BasicDBObject;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.IndexOptions;
public class Example {
public static void main(String[] args){
MongoCollection<Document> collection;
ObjectMapper objectMapper;
Document[] docArray;
List<Document> docList;
//Get the collection from the command line input and drop
collection = MongoClients.create("mongodb://example:password@localhost:27017/?authSource=admin").getCollection("Example");
String docListStr;
try {
String resourceFilePath = "C:\\sampledata\\example.json";
File sourceFile = new File(resourceFilePath);
if (!sourceFile.exists()) {
throw new IllegalArgumentException(String.format("The source file '%s' does not exist.", new Object[] { resourceFilePath }));
}
docListStr = FileUtils.readFileToString(sourceFile, StandardCharsets.UTF_8);
objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
docArray = objectMapper.readValue(docListStr, Document[].class);
docList = new ArrayList<>();
for (int i=0; i<docArray.length; i++) {
docList.add(docArray[i]);
}
collection.insertMany(docList);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
This works fine if I don't specify an object Id (one will be generated as expected) but I want to be able to clean this collection and re-insert documents with the same Ids as were present before. In the JSON file, I am specifying the object Id alongside other data fields as below:
[{
"_id": {
"$oid": "63df25a1343a3278747d6398"
},
"example": "test1",
"_class": "com.example.Example"
},
{
"_id": {
"$oid": "63df25a1343a3278747d6398"
},
"example": "test2",
"_class": "com.example.Example"
}]
However, when I run this, I hit the error below:
java.lang.IllegalArgumentException: Invalid BSON field name $oid
at org.bson.AbstractBsonWriter.writeName(AbstractBsonWriter.java:532)
at org.bson.codecs.DocumentCodec.writeMap(DocumentCodec.java:198)
at org.bson.codecs.DocumentCodec.writeValue(DocumentCodec.java:182)
at org.bson.codecs.DocumentCodec.beforeFields(DocumentCodec.java:167)
at org.bson.codecs.DocumentCodec.writeMap(DocumentCodec.java:192)
at org.bson.codecs.DocumentCodec.encode(DocumentCodec.java:141)
at org.bson.codecs.DocumentCodec.encode(DocumentCodec.java:45)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:63)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:29)
at com.mongodb.operation.BulkWriteBatch$WriteRequestEncoder.encode(BulkWriteBatch.java:387)
at com.mongodb.operation.BulkWriteBatch$WriteRequestEncoder.encode(BulkWriteBatch.java:377)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:63)
at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:29)
at com.mongodb.internal.connection.BsonWriterHelper.writeDocument(BsonWriterHelper.java:75)
at com.mongodb.internal.connection.BsonWriterHelper.writePayload(BsonWriterHelper.java:59)
at com.mongodb.internal.connection.CommandMessage.encodeMessageBodyWithMetadata(CommandMessage.java:143)
at com.mongodb.internal.connection.RequestMessage.encode(RequestMessage.java:138)
at com.mongodb.internal.connection.CommandMessage.encode(CommandMessage.java:57)
at com.mongodb.internal.connection.InternalStreamConnection.sendAndReceive(InternalStreamConnection.java:244)
at com.mongodb.internal.connection.UsageTrackingInternalConnection.sendAndReceive(UsageTrackingInternalConnection.java:99)
at com.mongodb.internal.connection.DefaultConnectionPool$PooledConnection.sendAndReceive(DefaultConnectionPool.java:444)
at com.mongodb.internal.connection.CommandProtocolImpl.execute(CommandProtocolImpl.java:72)
at com.mongodb.internal.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:200)
at com.mongodb.internal.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:269)
at com.mongodb.internal.connection.DefaultServerConnection.command(DefaultServerConnection.java:131)
at com.mongodb.operation.MixedBulkWriteOperation.executeCommand(MixedBulkWriteOperation.java:419)
at com.mongodb.operation.MixedBulkWriteOperation.executeBulkWriteBatch(MixedBulkWriteOperation.java:257)
at com.mongodb.operation.MixedBulkWriteOperation.access$700(MixedBulkWriteOperation.java:68)
at com.mongodb.operation.MixedBulkWriteOperation$1.call(MixedBulkWriteOperation.java:201)
at com.mongodb.operation.MixedBulkWriteOperation$1.call(MixedBulkWriteOperation.java:192)
at com.mongodb.operation.OperationHelper.withReleasableConnection(OperationHelper.java:424)
at com.mongodb.operation.MixedBulkWriteOperation.execute(MixedBulkWriteOperation.java:192)
at com.mongodb.operation.MixedBulkWriteOperation.execute(MixedBulkWriteOperation.java:67)
at com.mongodb.client.internal.MongoClientDelegate$DelegateOperationExecutor.execute(MongoClientDelegate.java:193)
at com.mongodb.client.internal.MongoCollectionImpl.executeInsertMany(MongoCollectionImpl.java:520)
at com.mongodb.client.internal.MongoCollectionImpl.insertMany(MongoCollectionImpl.java:504)
at com.mongodb.client.internal.MongoCollectionImpl.insertMany(MongoCollectionImpl.java:499)
How can I resolve this to allow for Object Ids to be spcified in the files?
Update:
I currently have a workaround in place, which is to check the keyset of each Document in the list, and if it matches "_id", fetch the corresponding value, parse it as a string to fetch the $oid attribute and convert to ObjectId. The existng "_id" key-value pair is then removed and added back with the ObjectId type instead:
for (Document docListEntry: docList) {
if (docListEntry.get("_id") != null) {
ObjectId objectId = new ObjectId(docListEntry.remove("_id").toString().split("}")[0].split("=")[1]);
docListEntry.put("_id",objectId);
}
}
It's not very clean, but it does resolve my issue for now
You can upgrade your mongo-driver version. I use following version works fine.
<!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>3.12.12</version>
</dependency>
And also you can use following code to get Document list easier.
CollectionType collectionType = objectMapper.getTypeFactory().constructCollectionType(ArrayList.class, Document.class);
List<Document> docList = objectMapper.readValue(docListStr, collectionType);
EDIT: After add $, you can use Document.parse instead of objectMapper.readValue, like this.
package com.example;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.CollectionType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import org.apache.commons.io.FileUtils;
import org.bson.Document;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
public class Example {
public static void main(String[] args) {
MongoCollection<Document> collection;
ObjectMapper objectMapper;
//Get the collection from the command line input and drop
MongoClient mongoClient = MongoClients.create("mongodb+srv://co:co@cluster0.tx0sb.mongodb.net/?retryWrites=true&w=majority");
collection = mongoClient.getDatabase("a").getCollection("Example");
String docListStr;
try {
String resourceFilePath = "C:\\sampledata\\example.json";
File sourceFile = new File(resourceFilePath);
if (!sourceFile.exists()) {
throw new IllegalArgumentException(String.format("The source file '%s' does not exist.", resourceFilePath));
}
docListStr = FileUtils.readFileToString(sourceFile, StandardCharsets.UTF_8);
objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
TypeFactory typeFactory = objectMapper.getTypeFactory();
CollectionType collectionType = typeFactory.constructCollectionType(ArrayList.class, typeFactory.constructMapType(LinkedHashMap.class, String.class, Object.class));
List<LinkedHashMap<String, Object>> mapList = objectMapper.readValue(docListStr, collectionType);
List<Document> docList = new ArrayList<>(mapList.size());
for (LinkedHashMap<String, Object> map : mapList) {
String docStr = objectMapper.writeValueAsString(map);
Document document = Document.parse(docStr);
docList.add(document);
}
collection.insertMany(docList);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}