NodeJS API,通过为每行调用多个API来上传和更新CSV文件

trnvg8h3  于 10个月前  发布在  其他
关注(0)|答案(1)|浏览(148)

我写了一个函数来更新CSV文件,通过调用多个API来获取所需的信息。
我的问题是,我可以做一个API来上传CSV文件,并调用这个函数到那个CSV文件。我想给予更新的文件作为对客户端的响应。如何才能实现或有其他方法吗?这里是我的主要功能:

async function processCSV(filePath) {
  try {
    const rows = []
    const uniqueReferenceNumbers = new Set()

    fs.createReadStream(filePath)
      .pipe(csv())
      .on('data', (row) => {
        const referenceNumber = row['Reference Number']
        console.log('reading from csv , referenceNumber:', referenceNumber)
        // Check if the reference number is unique before processing
        if (!uniqueReferenceNumbers.has(referenceNumber)) {
          uniqueReferenceNumbers.add(referenceNumber)
          rows.push(row)
        }
      })
      .on('end', async () => {
        if (rows.length === 0) {
          console.log('No unique reference numbers found in the CSV file.')
          return
        }

        // Assume that brand name is the same for all reference numbers in the CSV
        const brandName = rows[0]['Brand']

        // Make the first API call to get brand_uuid
        const brandApiResponse = await makeAPICallWithRetry(
          `https://api/v2/search/brand?q=${brandName}`
        )
        const brandUuid = brandApiResponse.data.data[0].uuid
        console.log('Brand uuid :', brandUuid)

        // Make the second, third, and fourth API calls for each unique reference number
        const updatedRows = [] // Accumulate modified rows in memory
        for (const row of rows) {
          const referenceNumber = row['Reference Number']

          try {
            // Make the second API call using the obtained brand_uuid and reference_number
            const secondApiResponse = await makeAPICallWithRetry(
              `https://api/v2/search/watch?q=${referenceNumber}&brand_uuid=${brandUuid}`
            )

            // Extract watch_uuid from the second API response
            const watchUuid = secondApiResponse.data.data[0].uuid
            console.log('watch uuid:', watchUuid)
            // Make the third API call to get product info
            const thirdApiResponse = await makeAPICallWithRetry(
              `https://api/v2/watch/info?uuid=${watchUuid}`
            )

            // Make the fourth API call to get product specs
            const fourthApiResponse = await makeAPICallWithRetry(
              `https://api/v2/watch/specs?uuid=${watchUuid}`
            )
            // Make the fifth API call to get price history
            const fifthApiResponse = await makeAPICallWithRetry(
              `https://api/v2/watch/price?uuid=${watchUuid}`
            )
            row['Product Info'] = thirdApiResponse
              ? JSON.stringify(thirdApiResponse.data, null, 2)
              : ''
            row['Product Specs'] = fourthApiResponse
              ? JSON.stringify(fourthApiResponse.data, null, 2)
              : ''
            row['Price History'] = fifthApiResponse
              ? JSON.stringify(fifthApiResponse.data, null, 2)
              : console.log(
                  `updating ${referenceNumber} with ${row['Product Info']}, ${row['Product Specs']} and ${row['Price History']} `
                )
            // Add the modified row to the accumulator
            updatedRows.push(row)
          } catch (error) {
            // Handle errors appropriately, e.g., log the error, skip the row, etc.
            console.error(
              `Error processing row for referenceNumber ${referenceNumber}: ${error.message}`
            )
            // Stop the entire process if the maximum number of retries is exceeded
            if (
              error.message.includes('Exceeded the maximum number of retries')
            ) {
              // Write accumulated data to the CSV before rethrowing the error
              if (updatedRows.length > 0) {
                const csvData = Papa.unparse(updatedRows, { header: true })
                fs.writeFileSync(filePath, csvData)
              }
              throw error
            }
          }
        }

        // Update the CSV file with the modified rows
        const csvData = Papa.unparse(rows, { header: true })
        fs.writeFileSync(filePath, csvData)

        // Process is complete, you can do something with the results
        console.log('API calls complete, CSV file updated')
      })
  } catch (error) {
    // Handle errors appropriately
    console.error(`Error processing CSV: ${error.message}`)
  }
}

字符串

o4tp2gmn

o4tp2gmn1#

我的问题是,我可以做一个API来上传CSV文件,并对该CSV文件调用此函数。我想给予更新的文件作为对客户端的响应。如何实现或有其他方法吗?
对于API,您可以使用J2EE. js和Multer库进行文件上传。这里有一个link教程,介绍如何设置它,但总结一下,API部分看起来像这样:

// Import libraries
const express = require('express');
const multer = require('multer');
const path = require('path');

// Initialise Express app
const app = express();
const port = 3000; // Port express will listen on

// Set up storage for uploaded files
const storage = multer.diskStorage({
  destination: (req, file, cb) => {
    cb(null, './uploads/');
  },
  filename: (req, file, cb) => {
    cb(null, Date.now() + '-' + file.originalname);
  }
});

// Create the multer instance
const upload = multer({
    storage: storage,
    });

// Set up a route for file uploads
app.post('/upload', upload.single('file'), (req, res) => {
    // Do the file processing here
    // Determine the file name and the absolute path
    let filename = req.file.filename;
    let filePath = path.resolve(`./uploads/${filename}`);
    // Process the file and send it as a response
    processCSV(filePath)
    res..sendFile(filePath);
    /*
    I'd suggest having the sendFile function inside a callback of the processCSV like so,
        processCSV(filePath, () => {
            res.status(200).sendFile(filePath);
        })
    Since the sendFile may get executed before processCSV due to the async nature of Node.js.
    */
});

app.listen(port, () => {
    console.log(`Server is running on port ${port}`);
});

字符串
js.js会监听端口3000和端点/upload。当一个论坛请求头为Content-Type: multipart/form-data,带有一个文件字段时,文件将以CurrentDateInUnixEpoch-OriginalFileName的命名格式写入。因此,上传的名为example.csv的文件将保存为1700644354000-example.csv。找到该文件的绝对路径(即相对于文件系统根目录的路径),并将其提供给您的函数。
正如我在代码的注解中提到的,你应该在processCSV()的回调中有res.sendFile(filePath),因为它可能会在你的函数完成之前被执行。
请记住,这只是一个起点。如果你想在生产环境或测试环境之外的任何地方使用它,你应该做更多的检查。测试,如输入大小检查,文件格式,检查等是必不可少的。记住,永远不要相信用户输入。
有关multer的更多信息,请参阅其NPM page

相关问题