if (typeof (awslambda) === 'undefined') { // For testing global.awslambda = require('./awslambda'); } const urlsFileName = process.env.PROXY_URLS_FILE_NAME const axios = require('axios'); const pipeline = require('util').promisify(require('stream').pipeline); const { defaultProvider } = require("@aws-sdk/credential-provider-node"); const { SignatureV4 } = require("@smithy/signature-v4"); const { Sha256 } = require("@aws-crypto/sha256-js"); const proxyUrls = require(`./${urlsFileName}`); exports.lambdaHandler = awslambda.streamifyResponse(async (event, responseStream, context) => { try { const random = Math.floor(Math.random() * proxyUrls.length); const proxyUrl = new URL(proxyUrls[random]); console.log('proxy-' + String(random) + ': ' + proxyUrl); let headers = Object.fromEntries( Object.entries(event.headers) .filter(([key]) => !key.toLowerCase().startsWith('x-amz') && !key.toLowerCase().startsWith('x-forwarded-') && key.toLowerCase() !== 'host') .map(([key, value]) => [key.toLowerCase() === 'authorization' ? 'lambda-scraper-' + key : key, value]) ); headers['host'] = proxyUrl.hostname; // URL encoding and decoding can be ambiguous (e.g. type=a&type=b becomes type=a,b) // Signing requires the query parameters to be encoded so we pass the raw query string in a header instead headers['lambda-scraper-raw-query-string'] = event.rawQueryString; const httpRequest = { method: event.requestContext.http.method, path: event.rawPath, // Needed for SignatureV4 url: proxyUrl + event.rawPath.substring(1), // query: event.queryStringParameters, // Needed for SignatureV4 // params: event.queryStringParameters, headers: headers, responseType: 'stream', timeout: 600 * 1000, // 10 minutes }; if (event.body) { httpRequest.body = event.body; // Needed for SignatureV4 httpRequest.data = event.body; } const credentials = await defaultProvider()(); const signer = new SignatureV4({ credentials: credentials, region: process.env.AWS_REGION, service: 'lambda', sha256: Sha256, }); const signedRequest = await signer.sign(httpRequest); let httpResponse; try { httpResponse = await axios(signedRequest); } catch (error) { if (error.response) { await pipeline( error.response.data, awslambda.HttpResponseStream.from(responseStream, { statusCode: error.response.status, headers: error.response.headers, }), ); return; } throw error; } headers = Object.fromEntries(Object.entries(httpResponse.headers).filter( ([key]) => !key.toLowerCase().startsWith('x-amz') )); await pipeline( httpResponse.data, awslambda.HttpResponseStream.from(responseStream, { statusCode: httpResponse.status, headers: headers, }), ); } catch (error) { console.error(error); responseStream = awslambda.HttpResponseStream.from(responseStream, { statusCode: 500, headers: { 'Content-Type': 'application/json', } }); responseStream.write(JSON.stringify({ message: error.message, stack: error.stack })); responseStream.end(); } }); if (require.main === module) { // For testing const event = { rawPath: '/https://ipinfo.io/ip', rawQueryString: '', headers: {}, requestContext: { http: { method: 'GET', } } }; exports.lambdaHandler(event, {}) .then() .catch(error => console.error(error)); }