c++ 为什么我的HTTPS请求在大文件上断开连接？

我正在使用HTTP 1通过HTTPS发出GET请求。1通过 * 以太网连接 *（此处不涉及WiFi）。
代码在下面。
对于非常小的文件（25B），文件被成功下载。但是当文件大小增加（20K或2MB）时不下载，连接会很快关闭。
所以我的问题是如何下载“大”文件？
我的代码运行在ESP32上。

#include <EthernetUdp.h>

#include <EthernetLarge.h>
#include <SSLClient.h>

// Choose the analog pin to get semi-random data from for SSL
// Pick a pin that's not connected or attached to a randomish voltage source
const int rand_pin = A5;
 
// Initialize the SSL client library
// We input an EthernetClient, our trust anchors, and the analog pin
EthernetClient base_client;
SSLClient client(base_client, TAs, (size_t)TAs_NUM, rand_pin);

void NanoEthernet::httpsGet(
    char const *hostname, char const *path, 
    byte *buffer, size_t bufferCapacity, 
    size_t &bodyLength, bool &bufferOverflows,
    void (*downloadChunkCallback)(byte * buffer, size_t bufferReadCount, size_t contentLength)
    ) {

    log("Time is: %d %d %d %02d:%02d:%02d", day(), month(), year(), hour(), minute(), second());

    char const * subModule = "httpsGet: ";
    log("%sbegin %s%s…",subModule, hostname, path);
    
    // === if you get a connection, report back via serial:
    if (!client.connect(hostname, 443)) {
        // if you didn't get a connection to the server:
        error("%sconnection failed", subModule);
        return false;
    }


    success("%sConnected!",subModule);
    // Make a HTTP request:
    client.printf("GET %s HTTP/1.1", path);             client.println();
    client.printf("Host: %s", hostname);                client.println();
    // client.println("Connection: keep-alive");            client.println();
    // client.println("Keep-Alive: timeout=5, max=100");    client.println();
    //client.println("Connection: close");
    client.println();

    //while(client.connected() && !client.available()) delay(1); //waits for data

    // success("%sSent query headers (connected:%d)",
    //  subModule,
    //  client.connected()
    // );

    
    /// Body counter
    int bodyCounter = 0;

    /// Chunk counter
    int chunkReadCount = 0;

    /// Whether to read response header?
    bool readResponseHeader = true;
    byte *p = buffer;

    int contentLength = 0;

    const unsigned timeoutNoData = 10*1000; // 10 seconds
    unsigned long long lastTimeNoData = millis();

    unsigned waitForConnection = 10;

    while (true) {
        // log("%sResponse: connected: %d | readResponseHeader: %d", 
        //  subModule,
        //  client.connected(),
        //  readResponseHeader
        // );
        if (!client.connected()) {
            delay(1);
            error("%sNo more connected, aborting", subModule);
            if (waitForConnection-- <= 0)
                continue;
            else
                break;
        }

        if (!client.available()) {
            delay(1);

            // == Still reading the header then continue, otherwise hanle timeout
            if (!readResponseHeader) {
                if (millis() - lastTimeNoData >= timeoutNoData) {
                    if (contentLength==0) {
                        log("%sNo more data are available after %d seconds timeout", 
                            subModule,
                            timeoutNoData/1000
                        );
                    }
                    else { 
                        error("%sNo more data are available timeout", subModule);
                    }
                    // == Stopping the connection
                    break;
                }
            }

            //log(">.");
            continue;
        }   
        lastTimeNoData = millis();

    
        // = Skip response header
        if (readResponseHeader) {
            const String header = client.readStringUntil('\n'); // End of line is "\r\n"

            log("%sResponse headers: %s", subModule, header.c_str());

            const int nextData = client.peek();
            // End of stream?
            if (nextData==-1) {
                error("%sEnd of stream (received data -1) while reading headers");
                break;
            }

            const char nextChar = char(nextData);
            //log("Response headers: => next char is %d 0x%02x (%c)", nextChar, nextChar, nextChar);
            // == End of headers? (empty line)      
            if (nextChar == '\r') {
                // Actually read the end of line
                client.read();  // eat \r
                client.read();  // eat \n
                //log("End of response header, starting to read response body…");
                readResponseHeader=false;
            }
            else {

                char const *headerString = "Content-Length: ";
                const int headerStringLength = 16;
                if (header.startsWith(headerString)) {
                    const String sContentLength = header.substring(headerStringLength);
                    contentLength = header.substring(headerStringLength).toInt();
                    //log("Read Content-Length: string: '%s'| int: %d bytes", sContentLength.c_str(), byteCountToRead);
                }
            }
            continue;
        }

        const int data = client.read(); //gets byte from ethernet buffer
        // == End of stream?
        if (data==-1) {
            log("%sEnd of stream (data received: -1)",subModule);
            break;
        }

        // == Keep the actual byte
        const byte b = byte(data);

        #if LOG_LEVEL >= LOG_LEVEL_DEBUG
        //log(">%02x (%d)", b, bodyCounter);
        log(">%02x ", b);
        #endif

        *p++ = b;
        bodyCounter++;
        chunkReadCount++;

        // = When no chunk callback, overflow means stop reading
        if (downloadChunkCallback==NULL) {
            // == If buffer overflows, stop
            if (bodyCounter >= bufferCapacity) {

                bufferOverflows = true;
                break;
            }
        }
        // = But when chunk callback has been provided, pass it the 
        // chunk, rewind the buffer and continue
        else {
                // == If buffer overflows, stop
            if (chunkReadCount >= bufferCapacity) {
                downloadChunkCallback(buffer, chunkReadCount, contentLength);
                // Now rewind the chunk to the satrt of the buffer prior to 
                // continue reading the reponsse body
                chunkReadCount = 0;
                p = buffer;
            }
        }

        // == We read everything?
        if (contentLength>0) {
            if (bodyCounter>=contentLength) {
                log("%sEnd of stream (received ontentLength bytes)", subModule);
                break;
            }
        }
    }
    

    // = When chunk callback, pass it the potential remaining chunk
    if(downloadChunkCallback!=NULL 
    && chunkReadCount>0) {
        downloadChunkCallback(buffer, chunkReadCount, contentLength);
    }

    // == Write the length we read
    bodyLength = bodyCounter;
    
    #if LOG_LEVEL >= LOG_LEVEL_DEBUG
    log("");
    log("%s: did the get and read %d bytes", subModule, bodyCounter);
    #endif
    
    client.stop();
    return true;    
}

我终于找到了解决办法。
关键是，当然，我没有2 MB的空闲内存（实际上ESP 32是一个320 KB RAM微控制器），所以关键是要求服务器发送一系列非常小的块，我们处理这些块，以连续的方式将它们追加到闪存上的文件中。
我想到的第一个候选者是chunk transfert encoding/，但是唉，由服务器来选择这样的模式，因为远程服务器实际上是Azure存储，所以没有提出hunk模式。这就是我调用回Range模式的原因：您只需在报头中指定Range: bytes: -以接收相应的数据位。因此，我最终打开了HTTPS连接，然后，使用保持活动连接，我制作了一系列GET`，我要求数据的滑动窗口。
代码片段如下：

/// @brief Writes the GET on [path] and write mandatory constant headers
/// @param host The host name
/// @param path The path of he document to get
static void httpsGetWriteMandatoryHeaders(char const *hostname, char const * path) {
    client.printf("GET %s HTTP/1.1", path);             client.println();
    client.printf("Host: %s", hostname);                client.println();

    client.printf("Accept-encoding: gzip, deflate");    client.println();
    client.printf("Cache-control: no-cache, no-store, must-revalidate"); client.println();

}

bool NanoEthernet :: httpsGet(
    char const *hostname, char const *path, 
    byte *buffer, size_t bufferCapacity, 
    size_t &bodyLength, bool &bufferOverflows,
    void (*downloadChunkCallback)(byte * buffer, size_t bufferReadCount, size_t contentLength),
    bool verbose
    ) {

    log("Time is: %d %d %d %02d:%02d:%02d", day(), month(), year(), hour(), minute(), second());

    char const * subModule = "httpsGet: ";
    
    if (verbose) log("%sbegin %s%s…",subModule, hostname, path);
    
    // === if you get a connection, report back via serial:
    if (!client.connect(hostname, 443)) {
        // if you didn't get a connection to the server:
        error("%sconnection failed", subModule);
        return false;
    }

    unsigned chunkIndex = 0;

    success("%sConnected!",subModule);
    // Make a HTTP request:
    httpsGetWriteMandatoryHeaders(hostname, path);
    client.printf("Range: bytes=%d-%d", chunkIndex*bufferCapacity, (chunkIndex+1)*bufferCapacity);              client.println();

    #if COMPUTE_CHECKSUM
    // Reset checksum variable.
    checksum32 = 0;
    #endif
    // Counter for each chunck 
    int messageCounter = 0;
    /// Body counter
    int bodyCounter = 0;

    /// Chunk counter
    int chunkReadCount = 0;

    /// Whether to read response header?
    bool readResponseHeader = true;
    byte *p = buffer;

    // Length of single chunck
    int contentLength = 0; 

    // // Length of full body (File to transfer)
    // int bodyLength = 0; 

    const unsigned timeoutNoData = 10*1000; // 10 seconds
    unsigned long long lastTimeNoData = millis();

    unsigned waitForConnection = 10;

    while (true) {
        
        if (!client.connected()) {
            delay(1);
            error("%sNo more connected, aborting", subModule);
            if (waitForConnection-- <= 0)
                continue;
            else
                break;
        }

        if (!client.available()) {
            delay(1);

            // == Still reading the header then continue, otherwise hanle timeout
            if (!readResponseHeader) {
                if (millis() - lastTimeNoData >= timeoutNoData) {
                    if (contentLength==0) {
                        if (verbose) 
                        log("%sNo more data are available after %d seconds timeout", 
                            subModule,
                            timeoutNoData/1000
                        );
                    }
                    else { 
                        error("%sNo more data are available timeout", subModule);
                    }
                    // == Stopping the connection
                    break;
                }
            }

            //log(">.");
            continue;
        }   
        lastTimeNoData = millis();

    
        // = Skip response header
        if (readResponseHeader) {
            const String header = client.readStringUntil('\n'); // End of line is "\r\n"
            #if LOG_LEVEL >= LOG_LEVEL_DEBUG
            if((chunkIndex%20)==0){
                if (verbose) log("%sResponse headers: %s", subModule, header.c_str());
            }
            #endif  

            const int nextData = client.peek();
            // End of stream?
            if (nextData==-1) {
                error("%sEnd of stream (received data -1) while reading headers");
                break;
            }

            const char nextChar = char(nextData);
            //log("Response headers: => next char is %d 0x%02x (%c)", nextChar, nextChar, nextChar);
            // == End of headers? (empty line)      
            if (nextChar == '\r') {
                // Actually read the end of line
                client.read();  // eat \r
                client.read();  // eat \n
                //log("End of response header, starting to read response body…");
                readResponseHeader=false;
            }
            else {

                // == Repsonse code

                //HTTP/1.1 404 Not Found
                char const *headerString = "HTTP/1.1 ";
                size_t headerStringLength = 9;

                if (header.startsWith(headerString)) {
                    const String sCode = header.substring(headerStringLength);
                    const unsigned code = header.substring(headerStringLength).toInt();
                    switch (code/100) {
                        case 2:
                            if (verbose) success("Response code: %d", code);
                            continue;

                        // == All others are considered as errors
                        case 3: warn("Response code: code %d is not handled specifically", code);

                        default:
                            error("Response code not 2xx: code %d, extiting", code);
                            client.stop();
                            return false;
                    }

                    continue;
                }

                // == Content length

                headerString = "Content-Length: ";
                headerStringLength = 16;

                if (header.startsWith(headerString)) {
                    const String sContentLength = header.substring(headerStringLength);
                    contentLength = header.substring(headerStringLength).toInt();
                    //log("Read Content-Length: string: '%s'| int: %d bytes", sContentLength.c_str(), byteCountToRead);
                }
                else if (header.startsWith("Content-Range:") && bodyLength==0){
                    const String content_size = header.substring(header.indexOf("/")+1);
                    bodyLength = content_size.toInt();
                    if (verbose) log("body has a length of: %d bytes",bodyLength);
                }
            }
            continue;
        }

        const int data = client.read(); //gets byte from ethernet buffer
        // == End of stream?
        if (data==-1) {
            if (verbose) log("%sEnd of stream (data received: -1)",subModule);
            break;
        }

        // == Keep the actual byte
        const byte b = byte(data);

        #if LOG_LEVEL >= LOG_LEVEL_DEBUG
        log(">%02x ", b);
        #endif

        *p++ = b;
        bodyCounter++;
        chunkReadCount++;
        messageCounter++;
        
        #if COMPUTE_CHECKSUM
        checkSum32(data); // Updating checksum 32
        #endif

        // = When no chunk callback, overflow means stop reading
        if (downloadChunkCallback==NULL) {
            // == If buffer overflows, stop
            if (bodyCounter >= bufferCapacity) {

                bufferOverflows = true;
                break;
            }
        }
        // = But when chunk callback has been provided, pass it the 
        // chunk, rewind the buffer and continue
        else {
                // == If buffer overflows, stop
            if (chunkReadCount >= bufferCapacity) {
                downloadChunkCallback(buffer, chunkReadCount, bodyLength);
                // Now rewind the chunk to the satrt of the buffer prior to 
                // continue reading the reponsse body
                chunkReadCount = 0;
                p = buffer;

            }
        }

        // == We read everything?
        if (contentLength>0) {
            if (messageCounter>=contentLength) {
                #if LOG_LEVEL >= LOG_LEVEL_DEBUG
                    log("%sEnd of stream (received contentLength bytes)", subModule);
                #endif
                client.flush();

                /// Ask for the new chunk
                if(chunkIndex*bufferCapacity<bodyLength){
                    
                    ++chunkIndex;
                
                    if (verbose) log("Asking for new chunk #%d of %d bytes max", chunkIndex, bufferCapacity);
                    
                    httpsGetWriteMandatoryHeaders(hostname, path);
                    if ((chunkIndex+1)*bufferCapacity<bodyLength){
                        client.printf("Range: bytes=%d-%d", chunkIndex*bufferCapacity+1, (chunkIndex+1)*bufferCapacity);        client.println();
                    }
                    else{
                        client.printf("Range: bytes=%d-%d", chunkIndex*bufferCapacity+1, bodyLength);       client.println();
                    }

                    client.println();
                    readResponseHeader=true;
                    messageCounter = 0;
                    while (!client.available()&&client.connected()) delay(1);
                }
                else{
                    if (verbose) success("All the data is well received");
                    break;
                }
            }
        }
    }
    

    // = When chunk callback, pass it the potential remaining chunk
    if(downloadChunkCallback!=NULL 
    && chunkReadCount>0) {
        downloadChunkCallback(buffer, chunkReadCount, bodyLength);
    }

    // == Write the length we read
    bodyLength = bodyCounter;
    
    // #if LOG_LEVEL >= LOG_LEVEL_DEBUG
    log("");
    log("%s: did the get and read %d bytes", subModule, bodyCounter);
    // #endif
    
    client.stop();
    
    #if COMPUTE_CHECKSUM
    if (verbose) log("------------- The checksum32 is %d -----------------", checksum32);
    #endif
    
    return true;    
}

c++ 为什么我的HTTPS请求在大文件上断开连接？

1条答案

相关问题

热门标签

最新问答