var assert = require('assert');
var archiver = require('archiver');
var async = require('async');
var AWS = require('aws-sdk');
var fs = require('fs');
var s3 = require('@auth0/s3');
function S3Zipper(awsConfig) {
var self = this
assert.ok(awsConfig, 'AWS S3 options must be defined.');
assert.notEqual(awsConfig.accessKeyId, undefined, 'Requires S3 AWS Key.');
assert.notEqual(awsConfig.secretAccessKey, undefined, 'Requires S3 AWS Secret');
assert.notEqual(awsConfig.region, undefined, 'Requires AWS S3 region.');
assert.notEqual(awsConfig.bucket, undefined, 'Requires AWS S3 bucket.');
if(awsConfig.sessionToken) {
AWS.config.update({
accessKeyId: awsConfig.accessKeyId,
secretAccessKey: awsConfig.secretAccessKey,
sessionToken: awsConfig.sessionToken,
region: awsConfig.region
}); }
else {
AWS.config.update({
accessKeyId: awsConfig.accessKeyId,
secretAccessKey: awsConfig.secretAccessKey,
region: awsConfig.region
}
self.init(awsConfig);
}
S3Zipper.prototype = {
init: function (awsConfig) {
this.awsConfig = awsConfig;
var self = this
self.s3bucket = new AWS.S3({
params: {
Bucket: self.awsConfig.bucket
}
});
}
, filterOutFiles: function (fileObj) {
return fileObj;
}
, calculateFileName: function (f) {
var name = f.Key.split("/");
name.shift();
name = name.join("/");
return name;
}
/*
params={
folderName: the name of the folder within the s3 bucket
, startKey: the key of the file you want to start after. keep null if you want to start from the first file
, maxFileCount: an integer that caps off how many files to zip at a time
, maxFileSize: max total size of files before they are zipped
, recursive: option to loop through nested folders
};
callback = function that is called back when completed
* */
, getFiles: function (params,callback) {
if(arguments.length == 5){ //for backwards comparability
params={
folderName: arguments[0]
, startKey: arguments[1]
, maxFileCount: arguments[2]
, maxFileSize: arguments[3]
, recursive:false
};
callback = arguments[4];
}
var bucketParams = {
Bucket: this.awsConfig.bucket, /* required */
Delimiter: "/",
Prefix: params.folderName + "/"
};
if (params.startKey)
bucketParams.Marker = params.startKey;
if (typeof(params.maxFileCount) == "function" && typeof(callback) == "undefined") {
callback = params.maxFileCount;
params.maxFileCount = null;
}
else if (params.maxFileCount > 0)
bucketParams.MaxKeys = params.maxFileCount;
var t = this;
var files ={};
files.Contents = [];
var options = {
s3Client: this.s3bucket
// more options available. See API docs below.
};
var client = s3.createClient(options);
var realParams = {
s3Params: bucketParams,
recursive: params.recursive
};
var emitter = client.listObjects(realParams);
emitter.on('data', function (data) {
if(data && data.Contents) {
files.Contents = files.Contents.concat(data.Contents);
}
});
emitter.on('error', function(err) {
console.error('unable to get files:', err.stack);
callback(err);
});
emitter.on('end', function () {
var data = files;
console.log('end');
var result = [];
var totalSizeOfPassedFiles = 0;
var lastScannedFile;
for (var i = 0; i < data.Contents.length; i++) {
var passedFile = t.filterOutFiles(data.Contents[i]);
if (passedFile) {
if (params.maxFileSize && params.maxFileSize < passedFile.Size) {
console.warn('Single file size exceeds max allowed size', data.Contents[i].Size, '>', params.maxFileSize, passedFile);
if (result.length == 0) {
console.warn('Will zip large file on its own', passedFile.Key);
result.push(passedFile);
totalSizeOfPassedFiles += passedFile.Size;
}
else
break;
}
else if (params.maxFileSize && totalSizeOfPassedFiles + data.Contents[i].Size > params.maxFileSize) {
console.log('Hit max size limit. Split fragment');
break;
}
else {
result.push(passedFile);
totalSizeOfPassedFiles += passedFile.Size;
}
}
lastScannedFile = data.Contents[i];
}
callback(null, {files: result, totalFilesScanned: data.Contents.length, lastScannedFile: lastScannedFile});
});
}
/*
params: {
pipe : pipe stream
, folderName: folder name to zip
, startKey: the key of the file you want to start after. keep null if you want to start from the first file
, maxFileCount: an integer that caps off how many files to zip at a time
, maxFileSize: max total size of files before they are zipped
, recursive: option to loop through nested folders
}
, callback : function
*/
, streamZipDataTo: function (params, callback) {
if (!params || !params.folderName) {
console.error('folderName required');
return null;
}
var zip = new archiver.create('zip');
if (params.pipe) zip.pipe(params.pipe);
var t = this;
this.getFiles(params, function (err, clearedFiles) {
if (err)
console.error(err);
else {
var files = clearedFiles.files;
console.log("files", files);
async.map(files, function (f, callback) {
t.s3bucket.getObject({Bucket: t.awsConfig.bucket, Key: f.Key}, function (err, data) {
if (err)
callback(err);
else {
var name = t.calculateFileName(f);
if (name === ""){
callback(null, f);
return;
}
else {
console.log('zipping ', name, '...');
zip.append(data.Body, {name: name});
callback(null, f);
}
}
});
}, function (err, results) {
zip.manifest = results;
zip.on('finish',function(){
callback(err, {
zip: zip,
zippedFiles: results,
totalFilesScanned: clearedFiles.totalFilesScanned,
lastScannedFile: clearedFiles.lastScannedFile
});
});
zip.finalize();
});
}
});
}
, uploadLocalFileToS3: function (localFileName, s3ZipFileName, callback) {
console.log('uploading ', s3ZipFileName, '...');
var readStream = fs.createReadStream(localFileName);//tempFile
this.s3bucket.upload({
Bucket: this.awsConfig.bucket
, Key: s3ZipFileName
, ContentType: "application/zip"
, Body: readStream
})
.on('httpUploadProgress', function (e) {
var p = Math.round(e.loaded / e.total * 100);
if (p % 10 == 0)
console.log('upload progress', p, '%');
})
.send(function (err, result) {
readStream.close();
if (err)
callback(err);
else {
console.log('upload completed.');
callback(null, result);
}
});
}
//all these timeouts are because streams arent done writing when they say they are
/*
params: {
s3FolderName: the name of the folder within the S3 bucket
, startKey: the key of the file you want to start after. keep null if you want to start from the first file
, s3ZipFileName: the name of the file you to zip to and upload to S3
, tmpDir: specifies the directory of the temporal zip file, default is node_modules/aws-s3-zipper
, recursive: indicates to zip nested folders or not
}
, callback: function
*/
, zipToS3File: function (params, callback) {
if(arguments.length == 5){
// for backward compatibility
params = {
s3FolderName:arguments[0]
,startKey:arguments[1]
,s3ZipFileName:arguments[2]
,tmpDir:arguments[3]
,recursive: false
};
callback= arguments[4];
}
var t = this;
params.tmpDir = params.tmpDir?params.tmpDir+"/":""
params.zipFileName = params.tmpDir+'__' + Date.now() + '.zip';
if (params.s3ZipFileName.indexOf('/') < 0)
params.s3ZipFileName = params.s3FolderName + "/" + params.s3ZipFileName;
this.zipToFile(params, function (err, r) {
if (r && r.zippedFiles && r.zippedFiles.length) {
t.uploadLocalFileToS3( params.zipFileName, params.s3ZipFileName, function (err, result) {
callback(null, {
zipFileETag: result.ETag,
zipFileLocation: result.Location,
zippedFiles: r.zippedFiles
});
fs.unlinkSync(params.zipFileName);
});
}
else {
console.log('no files zipped. nothing to upload');
fs.unlinkSync(params.zipFileName);
callback(null, {
zipFileETag: null,
zipFileLocation: null,
zippedFiles: []
});
}
});
}
/*
params: {
s3FolderName: the name of the folder within the S3 bucket
, startKey: the key of the file you want to start after. keep null if you want to start from the first file
, s3ZipFileName: the name of the file you to zip to and upload to S3
, maxFileCount: an integer that caps off how many files to zip at a time
, maxFileSize: max total size of files before they are zipped
, tmpDir: specifies the directory of the temporal zip file, default is node_modules/aws-s3-zipper
, recursive: indicates to zip nested folders or not
}
, callback: function
*/
, zipToS3FileFragments: function (params, callback) {
if(arguments.length == 7){
// for backward compatibility
params = {
s3FolderName:arguments[0]
, startKey:arguments[1]
, s3ZipFileName:arguments[2]
, maxFileCount:arguments[3]
, maxFileSize:arguments[4]
, tmpDir:arguments[5]
, recursive: false
};
callback= arguments[6];
}
var t = this;
///local file
params.tmpDir = params.tmpDir?params.tmpDir+"/":""
params.zipFileName = params.tmpDir+'__' + Date.now() + '.zip';
if (params.s3ZipFileName.indexOf('/') < 0)
params.s3ZipFileName = params.s3FolderName + "/" + params.s3ZipFileName;
var finalResult;
var count = 0;
this.zipToFileFragments(params, function (err, result) {
if (err)
callback(err);
else {
finalResult = result;
if (!result || result.length == 0)
callback(null, result); /// dont need to wait for uploads
}
})
.onFileZipped = function (fragFileName, result) {
var s3fn = params.s3ZipFileName.replace(".zip", "_" + count + ".zip");
count++;
uploadFrag(s3fn, fragFileName, result);
};
var pendingUploads = 0;// prevent race condition
function uploadFrag(s3FragName, localFragName, result) {
pendingUploads++;
t.uploadLocalFileToS3(localFragName, s3FragName, function (err, uploadResult) {
if (uploadResult) {
result.uploadedFile = uploadResult;
console.log('remove temp file ', localFragName);
fs.unlinkSync(localFragName);
}
pendingUploads--;
if (pendingUploads == 0 && finalResult) {
callback(null, finalResult);
}
});
}
}
/*
params={
s3FolderName: the name of the folder within the s3 bucket
, startKey: the key of the file you want to start after. keep null if you want to start from the first file
, zipFileName: zip file name
, recursive: option to loop through nested folders
};
callback = function that is called back when completed
* */
, zipToFile: function (params, callback) {
if(arguments.length == 4){
// for backward compatibility
params = {
s3FolderName:arguments[0]
, startKey:arguments[1]
, zipFileName:arguments[2]
, recursive: false
};
callback= arguments[3];
}
var filestream = fs.createWriteStream(params.zipFileName);
this.streamZipDataTo({
pipe: filestream
,folderName: params.s3FolderName
, startKey: params.startKey
, maxFileCount: params.maxFileCount
, maxFileSize: params.maxFileSize
, recursive: params.recursive
}, function (err, result) {
setTimeout(function () {
callback(err, result);
filestream.close();
}, 1000);
});
}
/*
params: {
s3FolderName: the name of the folder within the S3 bucket
, startKey: the key of the file you want to start after. keep null if you want to start from the first file
, zipFileName: the name of the file you to zip to and upload to S3
, maxFileCount: an integer that caps off how many files to zip at a time
, maxFileSize: max total size of files before they are zipped
, recursive: indicates to zip nested folders or not
}
, callback: function
*/
, zipToFileFragments: function (params, callback) {
if(arguments.length == 6){
// for backward compatibility
params = {
s3FolderName:arguments[0]
, startKey:arguments[1]
, s3ZipFileName:arguments[2]
, maxFileCount:arguments[3]
, maxFileSize:arguments[4]
, recursive: false
};
callback= arguments[5];
}
var events = {
onFileZipped: function () {
}
};
var report = {
results: []
, errors: []
, lastKey: null
};
if (params.maxFileSize && params.maxFileSize < 1024)
console.warn('Max File Size is really low. This may cause no files to be zipped, maxFileSize set to ', params.maxFileSize);
if (params.zipFileName.indexOf(".zip") < 0)
params.zipFileName += ".zip";
var t = this;
function garbageCollector(fileStream, result, fragFileName) {
setTimeout(function () {
fileStream.close();
if (result.zippedFiles.length == 0) /// its an empty zip file get rid of it
fs.unlinkSync(fragFileName);
else
events.onFileZipped(fragFileName, result);
}, 1000); /// TODO: Zip needs a bit more time to finishing writing. I'm sure there is a better way
}
var counter = 0;
function recursiveLoop(startKey, fragFileName, callback) {
var fileStream = fs.createWriteStream(fragFileName);
t.streamZipDataTo({
pipe : fileStream
, folderName: params.s3FolderName
, startKey:startKey
, maxFileCount:params.maxFileCount
, maxFileSize:params.maxFileSize
, recursive: params.recursive }, function (err, result) {
if (err)
report.errors.push(err);
else {
if (result.zippedFiles.length > 0) {
report.results.push(result);
report.lastKey = result.zippedFiles[result.zippedFiles.length - 1].Key;
}
/// you may have not zipped anything but you scanned files and there may be more
if (result.totalFilesScanned > 0)
recursiveLoop(result.lastScannedFile.Key, params.zipFileName.replace(".zip", "_" + counter + ".zip"), callback);
else ///you're done time to go home
callback(err, result);
counter++;
/// clean up your trash you filthy animal
garbageCollector(fileStream, result, fragFileName);
}
});
}
recursiveLoop(params.startKey, params.zipFileName, function () {
if (report.errors.length > 0)
callback(report.errors, report.results);
else
callback(null, report.results);
});
return events;
}
};
module.exports = S3Zipper;