node.jselasticsearchelasticsearch-bulk-apindjson

NewLine error in Elasticsearch bulk API post request


I am trying to use the elasticsearch bulk api to insert multiple records into an index. My JSON looks something like this: request json

I am inserting a new line (\\n) at the end of the document but I'm still getting the newline error.

    Error: {
        "error": {
            "root_cause": [
                {
                    "type": "illegal_argument_exception",
                    "reason": "The bulk request must be terminated by a newline [\n]"
                }
            ],
            "type": "illegal_argument_exception",
            "reason": "The bulk request must be terminated by a newline [\n]"
        },
        "status": 400
    }


Solution

  • Based on my previous answer and https://stackoverflow.com/a/50754789/8160318:

    const AWS = require('aws-sdk');
    const creds = new AWS.EnvironmentCredentials('AWS');
    
    const INDEX_NAME = 'index_name';
    
    const esDomain = {
        region: 'us-east-1',
        endpoint: 'yoursearchdomain.region.amazonaws.com',
        index: 'myindex',
        doctype: 'mytype'
    };
    
    const endpoint = new AWS.Endpoint(esDomain.endpoint);
    const req = new AWS.HttpRequest(endpoint);
    
    
    const docs_as_body_params = JSON.parse(
        '[' +
        `{"index":{}} {"tags":["ab","cd"],"question":"test this","answer":"answer first"} {"index":{}} {"tags":["de","fg"],"question":"test second","answer":"answer second"}`.split(
            /(\s?{"index":{}} )/g
        )
        .filter(match => match.length)
        .filter((_, index) => index % 2 !== 0) +
        ']'
    );
    
    const bulk_body = [];
    docs_as_body_params.map((doc) => {
        bulk_body.push({
            index: {
                _index: INDEX_NAME,
                _id: doc.id || null
            }
        });
        bulk_body.push(doc);
    });
    
    /// THE MOST IMPORTANT PART -- getting to a valid ndjson
    const ndjson_payload = bulk_body.map(JSON.stringify).join('\n') + '\n'
    
    req.method = 'POST';
    req.path = '_bulk'
    req.region = esDomain.region;
    req.headers['presigned-expires'] = false;
    req.headers['Host'] = endpoint.host;
    req.headers['Content-Type'] = 'application/json';
    req.body = ndjson_payload;
    
    var signer = new AWS.Signers.V4(req, 'es');
    signer.addAuthorization(creds, new Date());
    
    var send = new AWS.NodeHttpClient();
    send.handleRequest(req, null, function (httpResp) {
        var respBody = '';
        httpResp.on('data', function (chunk) {
            respBody += chunk;
        });
        httpResp.on('end', function (chunk) {
            console.log('Response: ' + respBody);
            context.succeed('Lambda added document ' + doc);
        });
    }, function (err) {
        console.log('Error: ' + err);
        context.fail('Lambda failed with error ' + err);
    });