To Note:
While running with await in the for loop, everything executed as expected
When changed to a Promise.all() solution IOT adhere to an es-lint rule, the complaintNo for Manhattan ('1') is not returned
According to es-lint: https://eslint.org/docs/rules/no-await-in-loop
"performing an await as part of each operation is an indication that the program is not taking full advantage of the parallelization benefits of async/await"
But it can still be performed.
Since this functin is not speed dependent, await es-lint error is ignored.
const boroughMaxComplaintNums = {};
try {
for (let i = 1; i < 6; i += 1) {
const base = i * 1000000;
const query = `SELECT MAX(complaintNo) FROM NycDobComplaint WHERE complaintNo < ${base +
1000000} AND complaintNo > ${base - 1}`;
console.log('Query: ', query);
//const maxComplaint = await db.execute(query);
const maxComplaint = db.execute(query);
const maxComplaintNum = Object.values(maxComplaint[0][0]);
boroughMaxComplaintNums[i] = maxComplaintNum;
}
logger.info(`Max ComplaintNo Object\n`, boroughMaxComplaintNums);
} catch (err) {
logger.warn(err);
} finally {
logger.info('Terminating db connection');
await db.end();
}
for (let i = 1; i < 6; i += 1) {
const highNum = i * 1000000 + 1000000;
const soqlWhereStatement = `complaint_number<'${highNum}' AND complaint_number>='${boroughMaxComplaintNums[i]}'`;
console.log('SoQL $where Statement: ', soqlWhereStatement);
await publisher(
datasetIdentifier,
'nyc-dob-complaint',
'nyc-dob-violations-subscriber',
complaintsCleanRecord,
{
$where: soqlWhereStatement
}
);
}const maxComplaintPromiseArray = [];
const publishArray = [];
const publishPromises = await Promise.all(publishArray);
console.log(publishPromises);
// DB Query
try {
for (let i = 1; i < 6; i += 1) {
const base = i * 1000000;
const query = `SELECT MAX(complaintNo) FROM NycDobComplaint WHERE complaintNo < ${base +
1000000} AND complaintNo > ${base - 1}`;
console.log('Query: ', query);
const maxComplaint = db.execute(query);
maxComplaintPromiseArray.push(maxComplaint);
}
} catch (err) {
logger.warn(err);
} finally {
logger.info('Terminating db connection');
await db.end();
}
const maxComplaintNumResult = await Promise.all(maxComplaintPromiseArray);
console.log(maxComplaintNumResult);
// Parse the return maxComplaintNumber and Verify the results
for (let i = 0; i < 5; i += 1) {
maxComplaintNumResult[i] = Object.values(maxComplaintNumResult[i][0][0]);
console.log(maxComplaintNumResult[i]);
}
let maxComplaintNumsNew = maxComplaintNumResult.map((x, i) => {
return x[0][0];
});
console.log(maxComplaintNumsNew);
// PUBLISH
let publishPromiseArray = [];
for (let i = 0; i < 5; i += 1) {
const highNum = (i + 1) * 1000000 + 1000000;
const whereStatement = `complaint_number<'${highNum}' AND complaint_number>='${maxComplaintNumResult[i]}'`;
console.log('$where Statement: ', whereStatement);
const publishRecords = publisher(
datasetIdentifier,
'nyc-dob-complaint',
'nyc-dob-violations-subscriber',
complaintsCleanRecord,
{
$where: whereStatement
}
);
publishPromiseArray.push(publishRecords);
}
const publishResults = await Promise.all(publishPromiseArray);