NodeJS 问题:异步请求中的408状态传播问题

1cklez4t  于 11个月前  发布在  Node.js
关注(0)|答案(1)|浏览(138)

这里有一个问题,当在一个循环中使用get(get)时,例如,一次1000个域,如果至少有一个域收到408响应,那么所有后续域也将是408。
这个错误似乎适用于整个客户端。
有什么想法可以解决这个问题吗?
我尝试为每个请求设置超时(没有全局配置)-问题没有解决。
例如:解析1000个域,第一个域-状态成功,第二个域- 408和所有以下408立即。

  • 客户 *:
const got = require('got');
const Logger = require('./logger');

let iconvLib = require('iconv-lite');

class Client
{
    constructor(url, data = {}, timeout = 6000, headers = {}) {
        this.url = url;
        this.data = data;
        this.timeout = timeout;
        this.customHeaders = headers;
        this.response = {};
        this.logger = new Logger();
    }

    getJson() {
        return JSON.parse(this.getBody());
    }

    getBody() {
        let response = this.getResponse();

        const contentType = response.headers['content-type'];
        const matches = contentType.match(/charset=([^;]+)/);
        let encoding = matches ? matches[1] : 'utf8';

        if(encoding === 'windows-1251') {
            encoding = 'win1251';
        }

        let body = iconvLib.decode(Buffer.from(response.body), encoding);

        //console.log(body);

        return body;
    }

    getHeaders() {
        return this.getResponse().headers;
    }

    getStatusText() {
        return this.getResponse().statusMessage;
    }

    getStatus() {
        return this.getResponse().statusCode;
    }

    getResponse() {
        return this.response ?? {};
    }

    async head() {
        const config = this.getConfig();
        if(Object.keys(this.data).length !== 0) {
            config.searchParams = this.data;
        }

        try {
            this.response = await got.head(this.url, config);
        } catch(e) {
            this.response = this.handleErrors(e);
        }

        return this;
    }

    async post() {
        const config = this.getConfig();
        if(Object.keys(this.data).length !== 0) {
            config.body = this.data;
        }

        try {
            this.response = await got.post(this.url, config);
        } catch(e) {
            this.response = this.handleErrors(e);
        }

        return this;
    }

    async get() {
        const config = this.getConfig();
        let params = '';
        if(Object.keys(this.data).length !== 0) {
            const sign = this.url.includes('?') ? '&' : '?';
            params += sign + this.serialize(this.data);
        }

        config.responseType = 'buffer';

        try {
            this.response = await got.get(this.url + params, config);
        } catch(e) {
            this.response = this.handleErrors(e);
        }

        return this;
    }

    handleErrors(e) {
        const response = e.response ?? {};
        const errorsStatusMap = {
            'ERR_TOO_MANY_REDIRECTS': 310,
            'ERR_NON_2XX_3XX_RESPONSE': 403,
            'ECONNREFUSED': 404,
            'ECONNRESET': 404,
            'ENOTFOUND': 404,
            'ERR_INVALID_URL': 404,
            'ETIMEDOUT': 408,
            'EAI_AGAIN': 408,
            'Z_BUF_ERROR': 507,
            'Z_DATA_ERROR': 507,
            'EPROTO': 526,
            'EHOSTUNREACH': 0,
            'HPE_INVALID_HEADER_TOKEN': 0,
            'ENETUNREACH': 0,
        };

        if(e.code) {
            response.statusCode = errorsStatusMap[e.code];
        }

        if(e.name) {
            response.statusText = e.name;
        }

        if(!response.body) {
            response.body = '';
        }

        if(!response.statusCode && response.statusCode !== 0) {
            response.statusCode = 0;
        }

        return response;
    }

    getConfig() {
        return {
            timeout: {
                request: this.timeout,
            },
            retry: {
                limit: 0,
                maxRetryAfter: undefined,
            },
            https: {
                rejectUnauthorized: false
            },
            headers: Object.assign({
                'Connection': 'close',
                'User-Agent': this.getUserAgent(),
                'Sec-Ch-Ua-Mobile': '?0',
                'Sec-Ch-Ua-Platform': 'Windows',
                'Sec-Fetch-Dest': 'document',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'none',
                'Sec-Fetch-User': '?1',
                'Upgrade-Insecure-Requests': 1,
                'Dnt': 1
            }, this.customHeaders)
        };
    }

    getUserAgent() {
        return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36';
    }

    serialize(obj, prefix) {
        let str = [],
            p;
        for (p in obj) {
            if (obj.hasOwnProperty(p)) {
                const k = prefix ? prefix + '[' + p + ']' : p,
                    v = obj[p];
                str.push((v !== null && typeof v === 'object') ?
                    this.serialize(v, k) :
                    encodeURIComponent(k) + '=' + encodeURIComponent(v));
            }
        }

        return str.join('&');
    }
}

module.exports = Client;

字符串

  • 我如何使用客户端 *:
async makeHttpsRequest(domain) {
    const client = new Client('https://' + domain);
    await client.get();

    return client;
}

  • 并在我的解析器的init函数中使用此请求 *:
let parsers = [];
for(const domainItem of domainList) {
    parsers.push(new Parser(domainItem['domain'], domainItem['id'] ?? 0));
}

parsers = await Promise.all(parsers.map(parser => parser.init())); // <- I call makeHttpsRequest here in parser.init();

xxls0lw8

xxls0lw81#

408是超时。你可能一次发送了太多的请求。错开一点或者限制并发。例如看看p-limit包。
也要记住,一个被拒绝的promise会立即拒绝整个Promise.all promise。可以使用Promise.allSettled或者在makeHttpsRequest函数中添加一个try/catch

相关问题