I have some small four function Lambda, which aims to transfer files from SFTP to an S3 bucket.
If I trigger the code locally, then it runs to completion and file is transferred from SFTP to S3, repeatedly.
If I upload the code to Lambda, and trigger using the test event, the code execute but unfortunately in general, no file is transferred. I do not receive any errors. But I don't get the console.logs from within the s3.upload(), so I assume means it's some sort of execution time-out issue? How do I ensure that all the async await code executes before the Lambda terminates? (It did however work once, but I've been unable to repeat it!)
const Client = require("ssh2-sftp-client");
const sftpConfig = require("./config/sftp");
const AWS = require("aws-sdk");
const s3 = new AWS.S3();
const sftpDir = "/mysftp/source/directory";
const bucketName = 'myTargetS3Bucket';
function timestampLessThanADayOld(timestampMilliseconds) {
const currentTimestampMilliseconds = Date.now();
const timeDifferenceMilliseconds =
currentTimestampMilliseconds - timestampMilliseconds;
return timeDifferenceMilliseconds < 1 * 24 * 60 * 60 * 1000;
}
const transferFile = async (file, sftp, bucketName) => {
try {
const sftpFilePath = `${sftpDir}/${file.name}`;
console.log(`Attempting to transfer ${file.name}`);
const fileData = await sftp.get(sftpFilePath);
const s3Key = `Uploads/${file.name}`;
const s3Params = {
Bucket: bucketName,
Key: s3Key,
Body: fileData,
};
await s3.upload(s3Params, (err, data) => {
console.log("data: ",data);
if (err) {
console.error("Error: ", err);
} else {
console.log(`Upload succeeded: ${data.Location}`);
}
});
} catch (error) {
console.error("Error reading file: ", error);
}
};
const copyNewFiles = async () => {
console.log("Transfer initiated");
const sftp = new Client();
try {
await sftp.connect(sftpConfig);
const files = await sftp.list(sftpDir);
for (let i = 0; i < files.length; i++) {
if (timestampLessThanADayOld(files[i].modifyTime)) {
await transferFile(files[i], sftp, bucketName);
}
}
} catch (error) {
console.error("Error connecting to SFTP:", error);
} finally {
sftp.end();
}
console.log("Transfer complete");
};
module.exports.handler = async (event, context) => {
await copyNewFiles();
}
// if executing locally, I add a `copyNewFiles();` here to trigger it,
// and run it using `node sftpTransfer.js`
From CloudWatch here's the rough output of the event log:
2024-01-02T19:10:49.464+00:00 INIT_START Runtime Version: nodejs:14.v42 Runtime Version ARN: arn:aws:lambda:eu-north-1::runtime:xxxxxx
2024-01-02T19:10:50.234+00:00 START RequestId: 9eb64b6a-xxxxxx Version: $LATEST
2024-01-02T19:10:50.238+00:00 2024-01-02T19:10:50.238Z 9eb64b6a-xxxxxx INFO Transfer initiated
2024-01-02T19:10:53.319+00:00 2024-01-02T19:10:53.319Z 9eb64b6a-xxxxxx INFO Attempting to transfer MyFile.txt
2024-01-02T19:10:54.175+00:00 2024-01-02T19:10:54.175Z 9eb64b6a-xxxxxx INFO Transfer complete
2024-01-02T19:10:54.180+00:00 END RequestId: 9eb64b6a-xxxxxx
2024-01-02T19:10:54.180+00:00 REPORT RequestId: 9eb64b6a-xxxxxx Duration: 3944.60 ms Billed Duration: 3945 ms Memory Size: 1024 MB Max Memory Used: 89 MB Init Duration: 769.30 ms
Here's how I got it to work eventually.
Here's the handler code that is triggered on an timer event:
/**
* Lambda handler module.
* @module handler
*/
const { copyNewFiles } = require("./utils/sftp");
const { uploadToS3 } = require("./utils/aws");
/**
* Lambda handler function.
* @param {Object} event - AWS Lambda event object.
* @param {Object} context - AWS Lambda context object.
* @returns {Promise<void>}
*/
const handler = async (event, context) => {
await copyNewFiles(uploadToS3);
};
module.exports = { handler };
Here's the SFTP copy files code:
/**
* Module for handling file transfer from SFTP.
* @module utils/sftp
*/
const Client = require("ssh2-sftp-client");
const { timestampLessThanADayOld } = require("./timeTools.js");
const { connectionParams, sftpDir } = require("../config/sftp.js");
/**
* Transfers a file from SFTP.
* @param {Object} file - File object from SFTP directory listing.
* @param {Object} sftp - SSH2 SFTP client instance.
* @param {Function} transferCallback - Callback function to transfer the file.
* @returns {Promise<void>}
*/
async function transferFile(file, sftp, transferCallback) {
const sftpFilePath = `${sftpDir}/${file.name}`;
try {
const fileData = await sftp.get(sftpFilePath);
await transferCallback(file.name, fileData);
} catch (error) {
console.error("Error reading file: ", error);
}
}
/**
* Copies new files from SFTP.
* @param {Function} transferCallback - Callback function to transfer files.
* @returns {Promise<void>}
*/
async function copyNewFiles(transferCallback) {
console.log("Transfer initiated");
const sftp = new Client();
try {
await sftp.connect(connectionParams);
const files = await sftp.list(sftpDir);
for (let i = 0; i < files.length; i++) {
if (timestampLessThanADayOld(files[i].modifyTime)) {
await transferFile(files[i], sftp, transferCallback);
}
}
} catch (error) {
console.error("Error connecting to SFTP:", error);
} finally {
sftp.end();
}
console.log("Transfer complete");
}
module.exports = { copyNewFiles, transferFile };
and here's the S3 upload code:
/**
* Module for handling AWS S3 operations.
* @module utils/aws
*/
const { S3 } = require("@aws-sdk/client-s3");
const { Upload } = require("@aws-sdk/lib-storage");
const { bucketName, s3DestinationFolder } = require("../config/sftp");
const s3 = new S3();
/**
* Transfers a file to AWS S3.
* @param {string} fileName - The name of the file to transfer.
* @param {Buffer} fileData - The file data to upload.
* @returns {Promise<void>}
*/
async function uploadToS3(fileName, fileData) {
const s3Key = `${s3DestinationFolder}/${fileName}`;
const s3Params = {
Bucket: bucketName,
Key: s3Key,
Body: fileData,
};
try {
const rc = await new Upload({
client: s3,
params: s3Params,
}).done();
console.log(rc);
} catch (error) {
console.error("Error uploading to S3: ", error);
}
}
module.exports = { uploadToS3 };