i have a file almost 500 MB. that is been exported from firebase. now i want to add that data into mysql. here is the format of json file.
{
"103597192838847432678": {
"Galaxy A213": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A231": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A233": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
}
},
"103597192838847432679": {
"Galaxy A213": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A231": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A233": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
}
},
"103597192838847432690": {
"Galaxy A213": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A231": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A233": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
}
}
}
what is have tried is this.
const express = require("express");
const db = require("../config/db");
const fs = require('fs');
const JSONStream = require('JSONStream');
const filePath = "./src/data/data.json";
const router = express.Router();
router.post("/", checkServer);
router.get("/", checkServer);
router.put("/", checkServer);
router.delete("/", checkServer);
module.exports = router;
async function checkServer(req, res, next) {
const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' } );
//const parser = JSONStream.parse('*');
const parser = JSONStream.parse('*.*');
fileStream.pipe(parser);
parser.on('data', (jsonObject) => {
console.log(jsonObject);
});
parser.on('end', () => {
console.log('All JSON objects processed.');
});
res.json({
status: true,
successMessage: "Server is running Here",
});
}
the data that i have get in the jsonObject is
"Galaxy A231": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
}
it is missing the key of the object. how can i read the entire object like this.
"103597192838847432678": {
"Galaxy A213": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A231": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
},
"Galaxy A233": {
"battery_level": "61",
"connected_Wifi": "Mobile Data",
"email_id": "canilda28@gmail",
"last_Location_Lat": "11.8397285",
"last_Location_Lng": "-15.6554222",
"last_seen": "1697471866857",
"tokenKey":"dL2bf4bnRNGgEn"
}
}
Just pass the emitKey: true
parameter then you will get key
and value
pair object
More on DOC
async function checkServer(req, res, next) {
const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' } );
const parser = fileStream.pipe(JSONStream.parse([{emitKey: true}])); //here
parser.on('data', async (jsonObject) => {
parser.pause(); // this will pause the next read
const key = jsonObject.key;
const value = jsonObject.value;
await somefunction(key,value); // insert into db
console.log(key,value);
parser.resume(); // continue to read next object
});
parser.on('end', () => {
console.log('All JSON objects processed.');
});
res.json({
status: true,
successMessage: "Server is running Here",
});
}