javamongodbbson

Invalid BSON field name $oid when loading into Mongo collection from file


I am attempting to write a basic loader for a MongoDb collection which will read prepared documents from a file and insert them into a collection The class I am using is currently written as below:

package com.example;

import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

import org.apache.commons.io.FileUtils;
import org.bson.Document;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.Gson;
import com.mongodb.BasicDBObject;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.IndexOptions;

public class Example {

    public static void main(String[] args){
        MongoCollection<Document> collection;
        ObjectMapper objectMapper;
        Document[] docArray;
        List<Document> docList;

        //Get the collection from the command line input and drop
        collection = MongoClients.create("mongodb://example:password@localhost:27017/?authSource=admin").getCollection("Example");
        String docListStr;
        try {
            String resourceFilePath = "C:\\sampledata\\example.json";
            File sourceFile = new File(resourceFilePath);
            
            if (!sourceFile.exists()) {
                throw new IllegalArgumentException(String.format("The source file '%s' does not exist.", new Object[] { resourceFilePath }));
            }
            docListStr = FileUtils.readFileToString(sourceFile, StandardCharsets.UTF_8);
            
            objectMapper = new ObjectMapper();
            objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
            docArray = objectMapper.readValue(docListStr, Document[].class);
            docList = new ArrayList<>();
            for (int i=0; i<docArray.length; i++) {
                docList.add(docArray[i]);
            }

            collection.insertMany(docList);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
}

This works fine if I don't specify an object Id (one will be generated as expected) but I want to be able to clean this collection and re-insert documents with the same Ids as were present before. In the JSON file, I am specifying the object Id alongside other data fields as below:

[{
    "_id": {
        "$oid": "63df25a1343a3278747d6398"
    },
    "example": "test1",
    "_class": "com.example.Example"
},
{
    "_id": {
        "$oid": "63df25a1343a3278747d6398"
    },
    "example": "test2",
    "_class": "com.example.Example"
}]

However, when I run this, I hit the error below:

java.lang.IllegalArgumentException: Invalid BSON field name $oid
    at org.bson.AbstractBsonWriter.writeName(AbstractBsonWriter.java:532)
    at org.bson.codecs.DocumentCodec.writeMap(DocumentCodec.java:198)
    at org.bson.codecs.DocumentCodec.writeValue(DocumentCodec.java:182)
    at org.bson.codecs.DocumentCodec.beforeFields(DocumentCodec.java:167)
    at org.bson.codecs.DocumentCodec.writeMap(DocumentCodec.java:192)
    at org.bson.codecs.DocumentCodec.encode(DocumentCodec.java:141)
    at org.bson.codecs.DocumentCodec.encode(DocumentCodec.java:45)
    at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:63)
    at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:29)
    at com.mongodb.operation.BulkWriteBatch$WriteRequestEncoder.encode(BulkWriteBatch.java:387)
    at com.mongodb.operation.BulkWriteBatch$WriteRequestEncoder.encode(BulkWriteBatch.java:377)
    at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:63)
    at org.bson.codecs.BsonDocumentWrapperCodec.encode(BsonDocumentWrapperCodec.java:29)
    at com.mongodb.internal.connection.BsonWriterHelper.writeDocument(BsonWriterHelper.java:75)
    at com.mongodb.internal.connection.BsonWriterHelper.writePayload(BsonWriterHelper.java:59)
    at com.mongodb.internal.connection.CommandMessage.encodeMessageBodyWithMetadata(CommandMessage.java:143)
    at com.mongodb.internal.connection.RequestMessage.encode(RequestMessage.java:138)
    at com.mongodb.internal.connection.CommandMessage.encode(CommandMessage.java:57)
    at com.mongodb.internal.connection.InternalStreamConnection.sendAndReceive(InternalStreamConnection.java:244)
    at com.mongodb.internal.connection.UsageTrackingInternalConnection.sendAndReceive(UsageTrackingInternalConnection.java:99)
    at com.mongodb.internal.connection.DefaultConnectionPool$PooledConnection.sendAndReceive(DefaultConnectionPool.java:444)
    at com.mongodb.internal.connection.CommandProtocolImpl.execute(CommandProtocolImpl.java:72)
    at com.mongodb.internal.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:200)
    at com.mongodb.internal.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:269)
    at com.mongodb.internal.connection.DefaultServerConnection.command(DefaultServerConnection.java:131)
    at com.mongodb.operation.MixedBulkWriteOperation.executeCommand(MixedBulkWriteOperation.java:419)
    at com.mongodb.operation.MixedBulkWriteOperation.executeBulkWriteBatch(MixedBulkWriteOperation.java:257)
    at com.mongodb.operation.MixedBulkWriteOperation.access$700(MixedBulkWriteOperation.java:68)
    at com.mongodb.operation.MixedBulkWriteOperation$1.call(MixedBulkWriteOperation.java:201)
    at com.mongodb.operation.MixedBulkWriteOperation$1.call(MixedBulkWriteOperation.java:192)
    at com.mongodb.operation.OperationHelper.withReleasableConnection(OperationHelper.java:424)
    at com.mongodb.operation.MixedBulkWriteOperation.execute(MixedBulkWriteOperation.java:192)
    at com.mongodb.operation.MixedBulkWriteOperation.execute(MixedBulkWriteOperation.java:67)
    at com.mongodb.client.internal.MongoClientDelegate$DelegateOperationExecutor.execute(MongoClientDelegate.java:193)
    at com.mongodb.client.internal.MongoCollectionImpl.executeInsertMany(MongoCollectionImpl.java:520)
    at com.mongodb.client.internal.MongoCollectionImpl.insertMany(MongoCollectionImpl.java:504)
    at com.mongodb.client.internal.MongoCollectionImpl.insertMany(MongoCollectionImpl.java:499)

How can I resolve this to allow for Object Ids to be spcified in the files?

Update:

I currently have a workaround in place, which is to check the keyset of each Document in the list, and if it matches "_id", fetch the corresponding value, parse it as a string to fetch the $oid attribute and convert to ObjectId. The existng "_id" key-value pair is then removed and added back with the ObjectId type instead:

for (Document docListEntry: docList) {
        if (docListEntry.get("_id") != null) {
            ObjectId objectId = new ObjectId(docListEntry.remove("_id").toString().split("}")[0].split("=")[1]);
            docListEntry.put("_id",objectId);   
        }
}

It's not very clean, but it does resolve my issue for now


Solution

  • You can upgrade your mongo-driver version. I use following version works fine.

    <!-- https://mvnrepository.com/artifact/org.mongodb/mongo-java-driver -->
    <dependency>
        <groupId>org.mongodb</groupId>
        <artifactId>mongo-java-driver</artifactId>
        <version>3.12.12</version>
    </dependency>
    

    And also you can use following code to get Document list easier.

    CollectionType collectionType = objectMapper.getTypeFactory().constructCollectionType(ArrayList.class, Document.class);
    List<Document> docList = objectMapper.readValue(docListStr, collectionType);
    

    EDIT: After add $, you can use Document.parse instead of objectMapper.readValue, like this.

    package com.example;
    
    import com.fasterxml.jackson.databind.DeserializationFeature;
    import com.fasterxml.jackson.databind.ObjectMapper;
    import com.fasterxml.jackson.databind.type.CollectionType;
    import com.fasterxml.jackson.databind.type.TypeFactory;
    import com.mongodb.client.MongoClient;
    import com.mongodb.client.MongoClients;
    import com.mongodb.client.MongoCollection;
    import org.apache.commons.io.FileUtils;
    import org.bson.Document;
    
    import java.io.File;
    import java.io.IOException;
    import java.nio.charset.StandardCharsets;
    import java.util.ArrayList;
    import java.util.LinkedHashMap;
    import java.util.List;
    
    public class Example {
    
        public static void main(String[] args) {
            MongoCollection<Document> collection;
            ObjectMapper objectMapper;
    
    
            //Get the collection from the command line input and drop
            MongoClient mongoClient = MongoClients.create("mongodb+srv://co:co@cluster0.tx0sb.mongodb.net/?retryWrites=true&w=majority");
            collection = mongoClient.getDatabase("a").getCollection("Example");
            String docListStr;
            try {
                String resourceFilePath = "C:\\sampledata\\example.json";
                File sourceFile = new File(resourceFilePath);
    
                if (!sourceFile.exists()) {
                    throw new IllegalArgumentException(String.format("The source file '%s' does not exist.", resourceFilePath));
                }
                docListStr = FileUtils.readFileToString(sourceFile, StandardCharsets.UTF_8);
    
                objectMapper = new ObjectMapper();
                objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
                TypeFactory typeFactory = objectMapper.getTypeFactory();
                CollectionType collectionType = typeFactory.constructCollectionType(ArrayList.class, typeFactory.constructMapType(LinkedHashMap.class, String.class, Object.class));
                List<LinkedHashMap<String, Object>> mapList = objectMapper.readValue(docListStr, collectionType);
    
                List<Document> docList = new ArrayList<>(mapList.size());
                for (LinkedHashMap<String, Object> map : mapList) {
                    String docStr = objectMapper.writeValueAsString(map);
                    Document document = Document.parse(docStr);
                    docList.add(document);
                }
                collection.insertMany(docList);
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    }