node.jsredisnode-redisnodejs-streamredis-cache

How to store the readStream files of Node.js into Redis and also how to retrieve the stored readStream files from Redis?


I tried converting the readStream (Image) to string and then storing it in Redis. Then retrieving the string from Redis and converting it back to readStream. But it didn't work out.

function getFile(fileKey) {
  console.log(fileKey);
  const downloadParams = {
    Key: fileKey,
    Bucket: bucketName,
  };

  return s3.getObject(downloadParams).createReadStream();
}

exports.getFile = getFile;

For converting stream to string I'm using stream-to-string. It gets converted and stored in Redis.

const { getFile } = require("../s3");
const redis = require("redis");

const client = redis.createClient();

var toString = require("stream-to-string");

exports.getFileFromS3Controller = async (req, res) => {
  console.log(req.params);
  const path = req.params.path;
  const key = req.params.key;
  const readStream = getFile(path + "/" + key);

  toString(readStream).then(function (msg) {
    // Set data to Redis
    client.setex(key, 3600, msg);
  });

  readStream.pipe(res);
};

On Retrieving from the Redis I am not getting it.

const redis = require("redis");
const client = redis.createClient(null, null, { detect_buffers: true });
const Readable = require("stream").Readable;

// Cache middleware
function cache(req, res, next) {
  const { path, key } = req.params;

  client.get(key, (err, data) => {
    if (err) throw err;

    if (data !== null) {
      var s = new Readable();
      s.push(data);
      s.push(null);
      s.pipe(res);
    } else {
      next();
    }
  });
}

router.get("/:path/:key", cache, getFileFromS3Controller);

Solution

  • You are not calling next. Another mistake is that the stream is not being saved anywhere in the req so you can access later from the controller. From what I see you are writing it directly in res which is a problem because after this you cannot use res anymore to send anything else.

    Here it is the code (not tested)

    exports.getFileFromS3Controller = (req, res) => {
      if (req.fileStream) {
          req.fileStream.pipe(res);
          return
      }
    
      console.log(req.params);
      const path = req.params.path;
      const key = req.params.key;
      const readStream = getFile(path + "/" + key);
    
      toString(readStream).then(function (msg) {
          // Set data to Redis
          client.setex(key, 3600, msg);
    
          // Conver string to readable
          const readable = new Readable();
          readable.push(msg);
          readable.push(null);
          readable.pipe(res);
      });
    };
    
    function cache(req, res, next) {
        const { path, key } = req.params;
    
        client.get(key, (err, data) => {
            if (err) throw err;
    
            if (data !== null) {
                var s = new Readable();
                s.push(data);
                s.push(null);
    
                req.fileStream = s;
            }
    
            next();
        });
    }
    

    Edit I fixed a mistake in my answer because a Readable stream cannot be rewinded.