I am trying to parse an RSS feed using request js and feedparser-promised libraries. I am able to parse the feed using the below code.
import Bottleneck from 'bottleneck';
const feedparser = require('feedparser-promised');
const limiter = new Bottleneck({
maxConcurrent: 1,
minTime: 333,
});
const httpOptions = {
uri: val.sourcefeedurl,
resolveWithFullResponse: true,
method: 'GET',
pool: false,
headers: {
'If-None-Match': val.etag,
'If-Modified-Since': val.LastModified,
Connection: 'keep-alive',
ciphers: 'DES-CBC3-SHA',
},
};
const response = await limiter.schedule(() => feedparser.parse(httpOptions));
But since I use the feedparser-promised library I am not able to cache the etag and Last Modified from the response headers.
I tried modifying feedparser-promised like this
'use strict';
const request = require('request');
const feedParser = require('./feedParser');
const parse = (requestOptions, feedparserOptions) => {
const metaData = {};
return new Promise((resolve, reject) => {
request.get(requestOptions).on('error', reject).on('response', async resp => {
if (resp.statusCode === 304) {
reject('Source not modified');
} else if (resp.statusCode === 200) {
metaData.etagin = await resp.headers.etag;
metaData.LastModifiedin = await resp.headers['last-modified'];
metaData.LastModifiedLocal = await resp.headers['last-modified'];
// console.log(metaData);
}
}).pipe(feedParser(feedparserOptions).on('error', reject).on('response', resolve));
});
};
module.exports = {
parse
};
Below is the feedParser file
'use strict';
const FeedParserStream = require('feedparser');
module.exports = (feedparserOptions, metaData) => {
// console.log(metaData, 'herre');
const parsedItems = [];
const feedparser = new FeedParserStream(feedparserOptions);
// console.log(feedparser);
feedparser.on('readable', () => {
// console.log(resp);
let item;
while (item = feedparser.read()) {
parsedItems.push(item);
}
return parsedItems;
}).on('end', function next() {
this.emit('response', parsedItems);
});
return feedparser;
};
So my question is how do I return the response headers along with the parsedItems (as in the code) while resolving the promise.
Help is very much appreciated.
Pass the metaData on end like
'use strict';
const FeedParserStream = require('feedparser');
module.exports = (feedparserOptions, metaData) => {
// console.log(metaData, 'herre');
const parsedItems = [];
const feedparser = new FeedParserStream(feedparserOptions);
// console.log(feedparser);
feedparser.on('readable', () => {
// console.log(resp);
let item;
while (item = feedparser.read()) {
parsedItems.push(item);
}
return parsedItems;
}).on('end', function next() {
this.emit('response', { parsedItems, metaData });
});
return feedparser;
};
and your feed-parser promised as
'use strict';
const request = require('request');
const feedParser = require('./feedParser');
const parse = (requestOptions, feedparserOptions) => {
const metaData = {};
return new Promise((resolve, reject) => {
request.get(requestOptions).on('error', reject).on('response', async resp => {
if (resp.statusCode === 304) {
reject('Source not modified');
} else if (resp.statusCode === 200) {
metaData.etagin = await resp.headers.etag;
metaData.LastModifiedin = await resp.headers['last-modified'];
metaData.LastModifiedLocal = await resp.headers['last-modified'];
// console.log(metaData);
}
}).pipe(feedParser(feedparserOptions, metaData).on('error', reject).on('response', resolve));
});
};
module.exports = {
parse
};