这里有一个问题,当在一个循环中使用get(get)时,例如,一次1000个域,如果至少有一个域收到408响应,那么所有后续域也将是408。
这个错误似乎适用于整个客户端。
有什么想法可以解决这个问题吗?
我尝试为每个请求设置超时(没有全局配置)-问题没有解决。
例如:解析1000个域,第一个域-状态成功,第二个域- 408和所有以下408立即。
- 客户 *:
const got = require('got');
const Logger = require('./logger');
let iconvLib = require('iconv-lite');
class Client
{
constructor(url, data = {}, timeout = 6000, headers = {}) {
this.url = url;
this.data = data;
this.timeout = timeout;
this.customHeaders = headers;
this.response = {};
this.logger = new Logger();
}
getJson() {
return JSON.parse(this.getBody());
}
getBody() {
let response = this.getResponse();
const contentType = response.headers['content-type'];
const matches = contentType.match(/charset=([^;]+)/);
let encoding = matches ? matches[1] : 'utf8';
if(encoding === 'windows-1251') {
encoding = 'win1251';
}
let body = iconvLib.decode(Buffer.from(response.body), encoding);
//console.log(body);
return body;
}
getHeaders() {
return this.getResponse().headers;
}
getStatusText() {
return this.getResponse().statusMessage;
}
getStatus() {
return this.getResponse().statusCode;
}
getResponse() {
return this.response ?? {};
}
async head() {
const config = this.getConfig();
if(Object.keys(this.data).length !== 0) {
config.searchParams = this.data;
}
try {
this.response = await got.head(this.url, config);
} catch(e) {
this.response = this.handleErrors(e);
}
return this;
}
async post() {
const config = this.getConfig();
if(Object.keys(this.data).length !== 0) {
config.body = this.data;
}
try {
this.response = await got.post(this.url, config);
} catch(e) {
this.response = this.handleErrors(e);
}
return this;
}
async get() {
const config = this.getConfig();
let params = '';
if(Object.keys(this.data).length !== 0) {
const sign = this.url.includes('?') ? '&' : '?';
params += sign + this.serialize(this.data);
}
config.responseType = 'buffer';
try {
this.response = await got.get(this.url + params, config);
} catch(e) {
this.response = this.handleErrors(e);
}
return this;
}
handleErrors(e) {
const response = e.response ?? {};
const errorsStatusMap = {
'ERR_TOO_MANY_REDIRECTS': 310,
'ERR_NON_2XX_3XX_RESPONSE': 403,
'ECONNREFUSED': 404,
'ECONNRESET': 404,
'ENOTFOUND': 404,
'ERR_INVALID_URL': 404,
'ETIMEDOUT': 408,
'EAI_AGAIN': 408,
'Z_BUF_ERROR': 507,
'Z_DATA_ERROR': 507,
'EPROTO': 526,
'EHOSTUNREACH': 0,
'HPE_INVALID_HEADER_TOKEN': 0,
'ENETUNREACH': 0,
};
if(e.code) {
response.statusCode = errorsStatusMap[e.code];
}
if(e.name) {
response.statusText = e.name;
}
if(!response.body) {
response.body = '';
}
if(!response.statusCode && response.statusCode !== 0) {
response.statusCode = 0;
}
return response;
}
getConfig() {
return {
timeout: {
request: this.timeout,
},
retry: {
limit: 0,
maxRetryAfter: undefined,
},
https: {
rejectUnauthorized: false
},
headers: Object.assign({
'Connection': 'close',
'User-Agent': this.getUserAgent(),
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': 'Windows',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': 1,
'Dnt': 1
}, this.customHeaders)
};
}
getUserAgent() {
return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36';
}
serialize(obj, prefix) {
let str = [],
p;
for (p in obj) {
if (obj.hasOwnProperty(p)) {
const k = prefix ? prefix + '[' + p + ']' : p,
v = obj[p];
str.push((v !== null && typeof v === 'object') ?
this.serialize(v, k) :
encodeURIComponent(k) + '=' + encodeURIComponent(v));
}
}
return str.join('&');
}
}
module.exports = Client;
字符串
- 我如何使用客户端 *:
async makeHttpsRequest(domain) {
const client = new Client('https://' + domain);
await client.get();
return client;
}
型
- 并在我的解析器的init函数中使用此请求 *:
let parsers = [];
for(const domainItem of domainList) {
parsers.push(new Parser(domainItem['domain'], domainItem['id'] ?? 0));
}
parsers = await Promise.all(parsers.map(parser => parser.init())); // <- I call makeHttpsRequest here in parser.init();
型
1条答案
按热度按时间xxls0lw81#
408是超时。你可能一次发送了太多的请求。错开一点或者限制并发。例如看看
p-limit
包。也要记住,一个被拒绝的promise会立即拒绝整个
Promise.all
promise。可以使用Promise.allSettled
或者在makeHttpsRequest
函数中添加一个try
/catch
。