const batchCalls = _.chunk(calls, 100);
for (const batchCall of batchCalls) {
await Promise.all(batchCall.map(call => call())) // makes a hundred calls in series
}
/*
Request passed to Targer App (5 requests per seconds)
Get the response for each request and passed the response to Source App
maxSockets: The maximum number of connections allowed at any given time. A value of 0 will prevent anything from going out. A value of Infinity will provide no concurrency limiting.
maxSocketsPerHost:The maximum number of connections per host allowed at any given time. A value of 0 will prevent anything from going out. A value of Infinity will provide no per-host concurrency limiting.
rateLimit: The number of milliseconds to wait before each maxSocketsPerHost
*/
var queue1 = new RequestQueue({'maxSockets': 5, 'maxSocketsPerHost': 5, 'rateLimit': 1000}, {
item: function(input, done) {
request(input.url, function(error, response) {
input.res.send(response.body);
done();
});
},
end: function() {
console.log("Queue 1 completed!");
}
});
//To queue request - A for loop could be used to send multiple request
queue1.enqueue({'url': ''});
function chunk (items, size) {
const chunks = [];
items = [].concat(...items);
while (items.length) { chunks.push(items.splice(0, size)); }
return chunks;
}
async function ProcessDevice(device) {
// do your work here
}
// splice your items into chunks of 100, then process each chunk
// catching the result of each ProcessDevice in the chunk.map
// the results of the chunk are passed into the .then( )
// and you have a .catch( ) in case there's an error anywhere in the items
var jobArray = chunk(items,100);
for (let i = 0; i < jobArray.length; i++) {
Promise.allSettled(
jobArray[i].map(ja => ProcessDevice(ja)))
.then(function(results) { console.log("PromiseResults: " + results); })
.catch((err) => { console.log("error: " + err); });
}
5条答案
按热度按时间arknldoa1#
将批处理构建为
Promise
s的嵌套数组,然后在await
s为每个Promise.all
解析的循环中对每个批处理使用Promise.all
。字符串
bybem2ql2#
如果你使用的是lodash,你可以使用chunk来简化它,chunk会把一个数组划分成指定的最大大小的块
所以在你的情况下你可以这样使用它
变量调用(比如说550个数组)
字符串
6g8kf2rb3#
您可以很容易地将bluebird Promise's map与并发选项一起使用。在拾取下一批之前,这将处理并发中提到的最大记录数。示例:
第一个月
bbmckpt74#
limited-request-queue
可用于对请求进行排队。有一些选项可用于设置任何给定时间的最大连接数。下面是我们用来每秒发送5个请求的代码。此外,在任何给定时间内只会发送5个请求。limited-request-queue
字符串
kiayqfof5#
如果我没弄错的话,你可以使用项目的“数组”和Promise.all()方法(或者在你的情况下,.allSettled()只查看每个调用的结果),然后像这样处理其中的每个:
字符串