3
2
2023-12-08
(C) Questetra, Inc. (MIT License)
This item downloads the specified files on Amazon S3. You can download multiple files at once.
When you download multiple files, you should write one Object Key per line.
この工程は、Amazon S3 の指定ファイルをダウンロードします。一度に複数のダウンロードが可能です。複数ダウンロードする場合、データ項目では
1 行につき 1 つずつオブジェクトキーを書くようにしてください。
https://support.questetra.com/bpmn-icons/service-task-aws-s3-file-download/
https://support.questetra.com/ja/bpmn-icons/service-task-aws-s3-file-download/
iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAAXNSR0IArs4c6QAABE9JREFUWEfN
l3tMlWUcxz/v4SCXA38gtyPTiXo2m8YdEp3cUzGxOdw0E5tRNqlsMDGlRRE5kFJjqUDLKEJsYuCN
xD8qA5oMGeBhXpIdbiuRrORSkOKBt71vHuIcDnRU2un5893v8nl/z/f3e55HwMpLsDS/V1CQo6Pe
MR6EKBECAG/A5b5/D9AhQBOI5weVg+VdDQ2DlsT+VwDvBZFqpXJkpyiwFbC3JChwRxAp0OsVOR1X
v+uezGdSgLl+YVtB2CuAysLERmYiDICY2qatKZjIf0IAjX9EniiKSQ+T2NRHEIR83aWql83FMgug
8Q8vEUWenYrkhhiCwFHdpeqN4+BMP0zln1tSCaMKSHsuIORP5Z+bxhIRk8ZqYhRAUrvCdkT3sIKz
FFoS5sg9hcbQHaMAGt/wD0SBZEsDPYqdIJKra65OkWLIANKQcdCrfjPt85zMXcTGRGCjtGFo6B6f
lRwnfMkibv58i2073ubwwRw8Pdx4bUcGcbHRrIlbQfnpc2xcvwZnJxV37w6Rd7iYwuJSU947fyoH
XKVhJQNo/MISRIRiU6uQQF96evvRtXXw8YE9eKk9adG147NwPum79/Hum9txUqnI2nuQVSuiUXu6
09f/OwqFgoQtycya6SWH/PGnrnEFExA36bQ1R+4DhH8iQqKp1aJgf57bsJaZXmpEUcTe3o6S0lO8
tHkDpScqeGp5FPrhYWrrGglbEkJ9YzNKpZLVK2PQtXVypvJrjpVXMDAwfioLUKjTVr8gA8zzC2/k
7/k+uubMnkVBbha9fX18efIsq2JjUHu48Xp6Nvuz07n1y69Ms53GjZvduLtNZ4bak4MfFVF2upLl
0WGsfXoli58I5JuqC6SkZZqTTFOrtjrQAHB7zMEiG0vO72Wmcb6mls+/KCd3z1s4ONjz4qs72ZWS
xJLQICrOfUvXzW6eT1jHja5unkncxrKopVy+2iLrpCh/H38MDLA5KdUcQE+rtnq6AUA0Z7E7PZWV
yyLR6/VcbNDi4e5Kbl4hPgvmy1tTeKSU1vZOMtJSqG/UytU58P47LF0cwjRbW2739PJhwaccP3nW
bNO0aquFSQEepdUs8R0LMG4LUrdtoa5BS82Fi5bEMrKRxCkJVaqYnZ0dZacqydp3yDSO0RaME6HU
AW9sf0Uuf+mJr2Q1P8haHx/HDLUHi4IDaL58jez9eabu/4hQ42e+DVUqRzJ2JbMseinDwyNcu66j
SXuFKz+0UFNbP9pekl3Y4hCCA3zxffwx5s2ZTW9fP8fKzrDiyQhZB1Xf1xkBGLXhRIPI4KGZ603i
pnVEhoXi6uKCjY3CbDGkyXdd1yYnlioWvzoWZ2cVRUfLJh9EE41ic1mkaRcaEoC/z0KcVI6yia69
k+strXLLWriMR7HkZNXDSAKw+nEsQVj1QjIquCm8jJrqwdzl9P93Kf0vKvHA13IDhFUfJgYIqz7N
xorIao9TCyfbQ5v9BWbg5jDAzldXAAAAAElFTkSuQmCC
{
// 認証設定を作成し、指定
const keyAuth = httpClient.createAuthSettingToken('Access Key', key);
configs.putObject('conf_AccessKey', keyAuth);
const secretAuth = httpClient.createAuthSettingToken('Secret Access Key', secret);
configs.putObject('conf_SecretKey', secretAuth);
configs.put('conf_Region', region);
if (isFixed) {
configs.put('conf_Bucket', bucket);
configs.put('conf_Keys', keys);
} else {
const bucketDef = engine.createDataDefinition('バケット', 1, 'q_bucket', 'STRING_TEXTFIELD');
configs.putObject('conf_Bucket', bucketDef);
engine.setData(bucketDef, bucket);
const objectKeyDef = engine.createDataDefinition('オブジェクトキー', 2, 'q_keys', 'STRING_TEXTAREA');
configs.putObject('conf_Keys', objectKeyDef);
engine.setData(objectKeyDef, keys);
}
const filesDef = engine.createDataDefinition('ファイル', 3, 'q_files', 'FILE');
configs.putObject('conf_Files', filesDef);
engine.setData(filesDef, files);
return filesDef;
};
/**
* 異常系のテスト
* @param errorMsg
*/
const assertError = (errorMsg) => {
try {
main();
fail();
} catch (e) {
expect(e.message).toEqual(errorMsg);
}
};
/**
* リージョンコードの形式が不正 - ハイフンを含まない
*/
test('Region Code is invalid - no hyphens', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'invalidregioncode';
const bucket = 'abc-def.ghi';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Region Code is invalid.');
});
/**
* リージョンコードの形式が不正 - ハイフンの間の文字列が長すぎる
*/
test('Region Code is invalid - too many characters', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'eu-toomanycharacters-1';
const bucket = 'abc-def.ghi';
prepareConfigs(key, secret, region, null, null, false, null);
assertError('Region Code is invalid.');
});
/**
* バケット名をデータ項目で指定し、値が空
*/
test('Bucket Name is blank', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
prepareConfigs(key, secret, region, null, null, false, null);
assertError('Bucket Name is blank.');
});
/**
* バケット名が 2 文字以下
*/
test('Bucket Name is invalid - too short', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'a';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* バケット名が 64 文字以上
*/
test('Bucket Name is invalid - too long', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'a'.repeat(64);
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* バケット名に不正な文字が含まれる
*/
test('Bucket Name is invalid - contains invalid character', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc/def';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* バケット名の先頭がハイフン
*/
test('Bucket Name is invalid - starts with a slash', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = '-abc';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* バケット名の先頭がドット
*/
test('Bucket Name is invalid - starts with a dot', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = '.abc';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* バケット名の末尾がハイフン
*/
test('Bucket Name is invalid - ends with a slash', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = '123-';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* バケット名の末尾がドット
*/
test('Bucket Name is invalid - ends with a dot', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = '123.';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* バケット名にドットが連続する
*/
test('Bucket Name is invalid - adjacent dots', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc..efg';
prepareConfigs(key, secret, region, bucket, null, false, null);
assertError('Bucket Name is invalid.');
});
/**
* オブジェクトキーが長すぎる
*/
test('Object key is too long', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const longKey = 'あ'.repeat(Math.ceil(MAX_OBJECT_KEY_BYTES / 3)); //「あ」は 3 バイト
const keys = 'abc\n' // 1 つ目のオブジェクトキーは問題ない
+ longKey; // 2 つ目のオブジェクトキーが長すぎる
prepareConfigs(key, secret, region, bucket, keys, false, null);
assertError(`The object key "${longKey}" is too long. Object key must be less than ${MAX_OBJECT_KEY_BYTES} bytes.`);
});
/**
* オブジェクトキーが長すぎる
*/
test('Object key is invalid - ends with a slash', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const objectKey = 'testfolder/';
prepareConfigs(key, secret, region, bucket, objectKey, false, null);
assertError(`The object key "${objectKey}" is invalid. Object key must not end with a slash.`);
});
/**
* オブジェクトキーが多すぎて、HTTP リクエスト数の上限を超える
*/
test('Too many object keys', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const keys = 'path\n'.repeat(httpClient.getRequestingLimit() + 1);
prepareConfigs(key, secret, region, bucket, keys, false, null);
assertError('The number of object keys exceeds the limit.');
});
/**
* API リクエストのテスト
* @param {Object} request
* @param request.url
* @param request.method
* @param request.headers
* @param {String} region
* @param {String} bucket
* @param {String} objectKey
*/
const assertRequest = ({
url,
method,
headers
}, region, bucket, objectKey) => {
expect(url).toEqual(`https://${bucket}.s3.${region}.amazonaws.com/${encodeURIComponent(objectKey)}`);
expect(method).toEqual('GET');
expect(headers['x-amz-checksum-mode']).toEqual('ENABLED');
// Authorization ヘッダのテストは省略
};
/**
* ファイルのテスト
* @param file
* @param name
* @param contentType
* @param body
*/
const assertFile = (file, name, contentType, body) => {
expect(file.getName()).toEqual(name);
expect(file.getContentType()).toEqual(contentType);
const text = fileRepository.readFile(file, "UTF-8");
expect(text).toEqual(body);
};
/**
* API リクエストでエラー
*/
test('Fail to request', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const objectKey = 'testfolder/testfile1.txt';
prepareConfigs(key, secret, region, bucket, objectKey, false, null);
httpClient.setRequestHandler((request) => {
assertRequest(request, region, bucket, objectKey);
return httpClient.createHttpResponse(400, 'application/json', '{}');
});
assertError(`Failed to download "${objectKey}" from S3. status: 400`);
});
/**
* 成功
* オブジェクトキーをデータ項目で指定
* いくつかのオブジェクトはチェックサムを持つ
* ファイル型データ項目の値は空
*/
test('Success - objectKey set by data item - some objects have checksums', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const objectKeys = ['testfolder/testfile1.txt', 'テスト/テスト.html', 'テスト.txt'];
const contentTypes = ['application/json', 'text/html', 'text/plain'];
const bodies = ['{}', '', 'テスト'];
const filesDef = prepareConfigs(key, secret, region, bucket, objectKeys.join('\n'), false, null);
let reqCount = 0;
httpClient.setRequestHandler((request) => {
assertRequest(request, region, bucket, objectKeys[reqCount]);
const response = httpClient.createHttpResponse(200, contentTypes[reqCount], bodies[reqCount]);
if (reqCount === 1) {
// sha256 チェックサムを指定
const data = engine.createByteArray(bodies[reqCount]);
const checksum = base64.encodeToString(digest.sha256(data));
response.addHeader('x-amz-checksum-sha256', checksum);
} else if (reqCount === 2) {
// sha1 チェックサムを指定
const data = engine.createByteArray(bodies[reqCount]);
const checksum = base64.encodeToString(digest.sha1(data));
response.addHeader('x-amz-checksum-sha1', checksum);
}
reqCount++;
return response;
});
expect(main()).toEqual(undefined);
expect(reqCount).toEqual(objectKeys.length);
const files = engine.findData(filesDef);
expect(files.size()).toEqual(objectKeys.length);
assertFile(files.get(0), "testfile1.txt", contentTypes[0], bodies[0]);
assertFile(files.get(1), "テスト.html", contentTypes[1], bodies[1]);
assertFile(files.get(2), "テスト.txt", contentTypes[2], bodies[2]);
});
/**
* SHA-256 のチェックサムが一致しない
*/
test('Checksum does not match - SHA-256', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const objectKeys = ['testfolder/testfile1.txt', 'テスト/テスト.html', 'テスト.txt'];
const contentTypes = ['application/json', 'text/html', 'text/plain'];
const bodies = ['{}', '', 'テスト'];
const filesDef = prepareConfigs(key, secret, region, bucket, objectKeys.join('\n'), false, null);
let reqCount = 0;
httpClient.setRequestHandler((request) => {
assertRequest(request, region, bucket, objectKeys[reqCount]);
const response = httpClient.createHttpResponse(200, contentTypes[reqCount], bodies[reqCount]);
if (reqCount === 1) {
// sha256 チェックサムが、ファイルの内容と一致しない
response.addHeader('x-amz-checksum-sha256', 'invalidchecksum');
}
reqCount++;
return response;
});
assertError(`The checksum of the downloaded file "${objectKeys[1]}" does not match.`
+ ` s3: invalidchecksum, file: ${base64.encodeToString(digest.sha256(engine.createByteArray(bodies[1])))}`);
});
/**
* SHA-1 のチェックサムが一致しない
*/
test('Checksum does not match - SHA-1', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const objectKeys = ['testfolder/testfile1.txt', 'テスト/テスト.html', 'テスト.txt'];
const contentTypes = ['application/json', 'text/html', 'text/plain'];
const bodies = ['{}', '', 'テスト'];
const filesDef = prepareConfigs(key, secret, region, bucket, objectKeys.join('\n'), false, null);
let reqCount = 0;
httpClient.setRequestHandler((request) => {
assertRequest(request, region, bucket, objectKeys[reqCount]);
const response = httpClient.createHttpResponse(200, contentTypes[reqCount], bodies[reqCount]);
if (reqCount === 1) {
// sha256 チェックサムを指定
const data = engine.createByteArray(bodies[reqCount]);
const checksum = base64.encodeToString(digest.sha256(data));
response.addHeader('x-amz-checksum-sha256', checksum);
} else if (reqCount === 2) {
// sha1 チェックサムが、ファイルの内容と一致しない
response.addHeader('x-amz-checksum-sha1', 'invalidchecksum-xxx');
}
reqCount++;
return response;
});
assertError(`The checksum of the downloaded file "${objectKeys[2]}" does not match.`
+ ` s3: invalidchecksum-xxx, file: ${base64.encodeToString(digest.sha1(engine.createByteArray(bodies[2])))}`);
});
/**
* 成功
* オブジェクトキーをデータ項目で指定
* オブジェクトキーが多数である場合
* ファイル型データ項目の値は空
*/
test('Success - objectKey set by data item - many object keys', () => {
const key = 'key-12345';
const secret = 'secret-67890';
const region = 'ap-northeast-1';
const bucket = 'abc123-efg.hij';
const objectKeys = '\n\npath\n'.repeat(httpClient.getRequestingLimit()); // 途中に空行があっても無視される
const filesDef = prepareConfigs(key, secret, region, bucket, objectKeys, false, null);
let reqCount = 0;
httpClient.setRequestHandler((request) => {
assertRequest(request, region, bucket, 'path');
return httpClient.createHttpResponse(200, 'text/plain', `test${reqCount++}`);
});
expect(main()).toEqual(undefined);
expect(reqCount).toEqual(httpClient.getRequestingLimit());
const files = engine.findData(filesDef);
expect(files.size()).toEqual(httpClient.getRequestingLimit());
for (let i = 0; i < files.size(); i++) {
assertFile(files.get(i), "path", "text/plain", `test${i}`);
}
});
/**
* 成功
* オブジェクトキーを固定値で指定
* 既にファイル型データ項目にファイルが添付されている
*/
test('Success - object key set as fixed value', () => {
const key = 'key-99999';
const secret = 'secret-00000';
const region = 'us-east-2';
const bucket = 'bucket-name-999';
const objectKey = 'testfile4';
const contentType = 'application/octet-stream';
const body = 'ddddd';
const files = new java.util.ArrayList();
files.add(engine.createQfile('testfile1.txt', 'text/plain', 'aaaaa'));
files.add(engine.createQfile('testfile2.txt', 'text/plain', 'bbbbb'));
files.add(engine.createQfile('testfile3.txt', 'text/plain', 'ccccc'));
const filesDef = prepareConfigs(key, secret, region, bucket, objectKey, true, files);
let reqCount = 0;
httpClient.setRequestHandler((request) => {
assertRequest(request, region, bucket, objectKey);
reqCount++;
return httpClient.createHttpResponse(200, contentType, body);
});
expect(main()).toEqual(undefined);
expect(reqCount).toEqual(1);
const results = engine.findData(filesDef);
expect(results.size()).toEqual(4);
assertFile(results.get(3), objectKey, contentType, body);
});
/**
* 成功
* オブジェクトキーを固定値で指定
* S3 上でのファイル名が長く、QBPMS の制限長を超える場合
*/
test('Success - object key set as fixed value - long file name', () => {
const key = 'key-99999';
const secret = 'secret-00000';
const region = 'us-east-2';
const bucket = 'bucket-name-999';
const objectKey = 'a'.repeat(FILE_NAME_MAX_LENGTH) + '.txt';
const contentType = 'text/md';
const body = 'long long file name';
const filesDef = prepareConfigs(key, secret, region, bucket, objectKey, true, null);
let reqCount = 0;
httpClient.setRequestHandler((request) => {
assertRequest(request, region, bucket, objectKey);
reqCount++;
return httpClient.createHttpResponse(200, contentType, body);
});
expect(main()).toEqual(undefined);
expect(reqCount).toEqual(1);
const results = engine.findData(filesDef);
expect(results.size()).toEqual(1);
assertFile(results.get(0), 'a'.repeat(FILE_NAME_MAX_LENGTH - 3) + '...', contentType, body);
});
]]>