javascriptnode.jsstreamfeedparserrequestjs

How can I access the response headers of a request that is piped to a feedparser


I am trying to parse an RSS feed using request js and feedparser-promised libraries. I am able to parse the feed using the below code.

import Bottleneck from 'bottleneck';

const feedparser = require('feedparser-promised');
const limiter = new Bottleneck({
  maxConcurrent: 1,
  minTime: 333,
});

const httpOptions = {
      uri: val.sourcefeedurl,
      resolveWithFullResponse: true,
      method: 'GET',
      pool: false,
      headers: {
        'If-None-Match': val.etag,
        'If-Modified-Since': val.LastModified,
        Connection: 'keep-alive',
        ciphers: 'DES-CBC3-SHA',
      },
    };
const response = await limiter.schedule(() => feedparser.parse(httpOptions));

But since I use the feedparser-promised library I am not able to cache the etag and Last Modified from the response headers.

I tried modifying feedparser-promised like this

'use strict';

const request = require('request');

const feedParser = require('./feedParser');

const parse = (requestOptions, feedparserOptions) => {
  const metaData = {};
  return new Promise((resolve, reject) => {
    request.get(requestOptions).on('error', reject).on('response', async resp => {
      if (resp.statusCode === 304) {
        reject('Source not modified');
      } else if (resp.statusCode === 200) {
        metaData.etagin = await resp.headers.etag;
        metaData.LastModifiedin = await resp.headers['last-modified'];
        metaData.LastModifiedLocal = await resp.headers['last-modified'];
        // console.log(metaData);
      }
    }).pipe(feedParser(feedparserOptions).on('error', reject).on('response', resolve));
  });
};

module.exports = {
  parse
};

Below is the feedParser file

'use strict';

const FeedParserStream = require('feedparser');

module.exports = (feedparserOptions, metaData) => {
  // console.log(metaData, 'herre');
  const parsedItems = [];
  const feedparser = new FeedParserStream(feedparserOptions);
  // console.log(feedparser);
  feedparser.on('readable', () => {
    // console.log(resp);
    let item;
    while (item = feedparser.read()) {
      parsedItems.push(item);
    }

    return parsedItems;
  }).on('end', function next() {
    this.emit('response', parsedItems);
  });
  return feedparser;
};

So my question is how do I return the response headers along with the parsedItems (as in the code) while resolving the promise.

Help is very much appreciated.


Solution

  • Pass the metaData on end like

    'use strict';
    
    const FeedParserStream = require('feedparser');
    
    module.exports = (feedparserOptions, metaData) => {
      // console.log(metaData, 'herre');
      const parsedItems = [];
      const feedparser = new FeedParserStream(feedparserOptions);
      // console.log(feedparser);
      feedparser.on('readable', () => {
        // console.log(resp);
        let item;
        while (item = feedparser.read()) {
          parsedItems.push(item);
        }
    
        return parsedItems;
      }).on('end', function next() {
        this.emit('response', { parsedItems, metaData });
      });
      return feedparser;
    };
    

    and your feed-parser promised as

    'use strict';
    
    const request = require('request');
    
    const feedParser = require('./feedParser');
    
    const parse = (requestOptions, feedparserOptions) => {
      const metaData = {};
      return new Promise((resolve, reject) => {
        request.get(requestOptions).on('error', reject).on('response', async resp => {
          if (resp.statusCode === 304) {
            reject('Source not modified');
          } else if (resp.statusCode === 200) {
            metaData.etagin = await resp.headers.etag;
            metaData.LastModifiedin = await resp.headers['last-modified'];
            metaData.LastModifiedLocal = await resp.headers['last-modified'];
            // console.log(metaData);
          }
        }).pipe(feedParser(feedparserOptions, metaData).on('error', reject).on('response', resolve));
      });
    };
    
    module.exports = {
      parse
    };