javascriptnode.jsstreamreadlinelarge-files

Edit the final part of large(1.5gb) text file in NodeJS


My tool appends little json blocks with comma at their end into a txt file which initially has [ as the first character, in order to create a whole JSON format text file, like below,

{data: text1},

and txt file seems like at the end of the day,

[{data: text1},{data: text2},{data: text3},

So I have to remove last comma at the end of the text and put a ] to make it valid JSON right, but this file size is around 1.5GB so I couldn't figure how to edit it in NodeJS in order to make it valid JSON file.


Solution

  • const fs = require('fs');
    
    // Input and output file paths
    const inputFile = 'input.json';
    
    // Open the file for reading and writing
    fs.open(inputFile, 'r+', (err, fd) => {
      if (err) {
        console.error('Error opening the input file:', err);
        return;
      }
    
      // Move to the end of the file
      fs.fstat(fd, (err, stats) => {
        if (err) {
          console.error('Error reading file information:', err);
          fs.close(fd, () => console.log('Closed the input file due to error.'));
          return;
        }
    
        const endPosition = stats.size;
    
        // Create a buffer for reading
        const buffer = Buffer.alloc(1); // Read only one byte (comma)
    
        // Read the last byte in the file
        fs.read(fd, buffer, 0, buffer.length, endPosition - 1, (err, bytesRead, data) => {
          if (err) {
            console.error('Error reading the last byte:', err);
            fs.close(fd, () => console.log('Closed the input file due to error.'));
            return;
          }
    
          // Check if the last byte is a comma
          if (data[0] !== 44) { // Unicode code for comma is the same as ASCII
            console.log('The last character is not a comma.');
            fs.close(fd, () => console.log('Closed the input file.'));
            return;
          }
    
          // If the last byte is a comma, replace it with a closing bracket
          const closingBracket = Buffer.from(']', 'utf8'); // Use UTF-8 encoding
          fs.write(fd, closingBracket, 0, closingBracket.length, endPosition - 1, (err) => {
            if (err) {
              console.error('Error writing to the file:', err);
            } else {
              console.log('Replaced the comma with a closing bracket.');
            }
            // Close the input file
            fs.close(fd, () => console.log('Closed the input file after modification.'));
          });
        });
      });
    });