C++中使用curl库异步下载多个文件

vcirk6k6  于 2023-05-02  发布在  其他
关注(0)|答案(1)|浏览(264)

我想写一个程序,将下载一些文件异步。我设法使用curl_easy接口下载了一个文件,但由于某种原因,curl_multi接口根本不下载文件。我做错了什么?
下面是我的函数(download_file按预期工作,download_files只创建空文件,但不下载它们):
样品输入:

file_names = {"text1.txt","text2.txt","text3.txt"}; file_path = R"(C:\Downloads\\)";
url = "http://downloadPage.com/"
static size_t write_data(const char *ptr, size_t size, size_t nmemb, std::ofstream *userdata);

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url);

bool download_file(const std::string &file_name, const std::string &file_path, const char *url) {
    CURL *curl;
    CURLcode res;
    curl = curl_easy_init();
    if (curl) {
        static std::ofstream file_write((file_path + file_name), std::ios::binary);

        curl_easy_setopt(curl, CURLOPT_URL, url);
        curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
        curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
        curl_easy_setopt(curl, CURLOPT_WRITEDATA, &file_write);
        curl_easy_perform(curl);
        curl_easy_cleanup(curl);

        file_write.close();

    } else {
        std::cout << "curl error!" << std::endl;
    }
    return res;
}

void download_files(std::vector<std::string> file_names, const std::string &file_path, const char *url,
                    int HANDLECOUNT) {
    CURLM *multi_handle;
    CURLMsg *msg;
    unsigned int transfers = 0;
    int left = 0;
    int msgs_left = -1;
    int i = 0;

    curl_global_init(CURL_GLOBAL_ALL);
    multi_handle = curl_multi_init();

    curl_multi_setopt(multi_handle, CURLMOPT_MAXCONNECTS, 8);

    int still_running = 1; /* keep number of running handles */

    /* Allocate one CURL handle per transfer */
    for (transfers = 0; transfers < 8 && transfers < HANDLECOUNT; transfers++)
        add_transfer(multi_handle, transfers, &left, file_names, file_path, url);

    do {
        int still_alive = 1;
        curl_multi_perform(multi_handle, &still_alive);

        while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
            if (msg->msg == CURLMSG_DONE) {
                char *url2;
                CURL *e = msg->easy_handle;
                curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &url2);
                fprintf(stderr, "R: %d - %s <%s>\n",
                        msg->data.result, curl_easy_strerror(msg->data.result), url2);
                curl_multi_remove_handle(multi_handle, e);
                curl_easy_cleanup(e);
                left--;
            } else {
                fprintf(stderr, "E: CURLMsg (%d)\n", msg->msg);
            }
            if (transfers < HANDLECOUNT)
                add_transfer(multi_handle, transfers++, &left, file_names, file_path, url);
        }
        if (left)
            curl_multi_wait(multi_handle, NULL, 0, 1000, NULL);
    } while (left);
}

static size_t write_data(const char *ptr, size_t size, size_t nmemb, std::ofstream *userdata) {
    size_t nbytes = size * nmemb;
    userdata->write(ptr, nbytes);
    return nbytes;
}

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url) {
    CURL *eh = curl_easy_init();
    std::ofstream file_write((file_path + file_names[i]), std::ios::binary);
    std::cout << (file_path + file_names[i]) << " - " << (url + file_names[i]) << std::endl;
    curl_easy_setopt(eh, CURLOPT_URL, (url + file_names[i]).c_str());
    curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, write_data);
    curl_easy_setopt(eh, CURLOPT_WRITEDATA, &file_write);
    curl_multi_add_handle(cm, eh);
    (*left)++;
}
ssgvzors

ssgvzors1#

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url) {
    CURL *eh = curl_easy_init();
    std::ofstream file_write((file_path + file_names[i]), std::ios::binary);
    curl_easy_setopt(eh, CURLOPT_WRITEDATA, &file_write);
    // <...>
}

它存储在函数返回时销毁的局部变量file_write的地址。CURLOPT_WRITEDATA得到一个悬空指针。

static size_t write_data(const char *ptr, size_t size, size_t nmemb, std::ofstream *userdata) {
    size_t nbytes = size * nmemb;
    userdata->write(ptr, nbytes);
    return nbytes;
}

它使用悬空指针userdata,这导致未定义的行为。顺便说一句,回调函数的签名在最后一个参数中是错误的,这也会导致未定义的行为。
一个可能的解决方案是使用CURLOPT_PRIVATE curl easy选项:

static void
add_transfer(CURLM *cm, int i, int *left, std::vector<std::string> &file_names, const std::string &file_path,
             const char *url) {
    CURL *eh = curl_easy_init();
    auto *file_write = new std::ofstream ((file_path + file_names[i]), std::ios::binary);
    curl_easy_setopt(eh, CURLOPT_PRIVATE, file_write);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, file_write);
    // <...>
}

void download_files(std::vector<std::string> file_names, const std::string &file_path, const char *url,
                    int HANDLECOUNT) {
    // <...>
    do {
        // <...>
        while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
            if (msg->msg == CURLMSG_DONE) {
                // <...>
                curl_multi_remove_handle(multi_handle, e);
                void *file_write{};
                curl_easy_getinfo(e, CURLINFO_PRIVATE, &file_write);
                delete static_cast<std::ofstream*>(file_write);
                curl_easy_cleanup(e);
                left--;
            } else {
        // <...>
    } while (left);
}

相关问题