NodeJS 如何增强我的日志统计功能？

ecbunoof 于 2023-06-05 发布在 Node.js

关注(0)|答案(1)|浏览(194)

我正在节点上运行日志扫描和统计功能，由于文件大小有点大，所以执行时间很长，所以我尝试使用promise pool来增强它。下面是我的函数的简单工作流程。

const fs = require("fs");
const readline = require("readline");

const filePathList = [
  ".\\log1.log",
  ".\\log2.log",
  ".\\log3.log",
  ".\\log4.log",
  ".\\log5.log",
  ".\\log6.log",
  ".\\log7.log",
  ".\\log8.log",
  ".\\log9.log",
  ".\\log10.log",
];

const poolSize = 10;
let promisePool = new Set();

const statistic = async () => {
  while (filePathList.length > 0) {
    while (promisePool.size < poolSize) {
      const filePath = filePathList.shift();

      const promise = new Promise(async (resolve, reject) => {
        try {
          const fileStream = fs.createReadStream(filePath);
          const readLine = readline.createInterface({
            input: fileStream,
            crlfDelay: Infinity,
          });

          for await (const line of readLine) {
            // do something...
          }

          fileStream.destroy();
          promisePool.delete(promise);
          console.log(
            `${filePath} DONE!, remain promises: ${promisePool.size}`
          );

          resolve();
          analyzLog(threadIndex);
        } catch (err) {
          reject(err);
        }
      });
      promisePool.add(promise);
    }
    await Promise.race(promisePool);
  }
  await Promise.all(promisePool);
};

(async () => {
  const start = Date.now();
  console.log(`Start statistic algorithm, timestamp: ${start}`);
  await statistic();
  const end = Date.now();
  console.log(`End statistic algorithm, timestamp: ${end}`);
  console.log(`It take ${(end - start) / 1000} sec`);
})();

据我所知，文件系统是在节点外运行的，而且是多线程的。所以我希望它能快十倍。但无论我使用池大小10还是1运行它。它花费了类似的时间成本，所以我想我没有真正做分析的一部分同时进行。我怎么才能让它工作？

node.js

来源：https://stackoverflow.com/questions/76355234/how-to-boost-up-my-log-statistic-function

1条答案

按热度按时间

41zrol4v1#

你只能用Promise.all来实现这一点。将每个任务放入单个promise中，然后将其推入数组中，然后执行Promise.all(array)，以便所有任务并行运行。

const fs=require("fs");
const readline=require("readline");

const filePathList=[
  ".\\log1.log",
  ".\\log2.log",
  ".\\log3.log",
  ".\\log4.log",
  ".\\log5.log",
  ".\\log6.log",
  ".\\log7.log",
  ".\\log8.log",
  ".\\log9.log",
  ".\\log10.log",
];
// const poolSize=10;
const statistic=async () => {
  let promisePool=[];
  let fileLength=filePathList.length;
  filePathList.forEach((filePath) => {
    promisePool.push(new Promise(async (resolve, reject) => {
      let fileStream=fs.createReadStream(filePath);
      let readLine=readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity,
      });

      for await (let line of readLine) {
        // do something...
      }

      fileStream.destroy();
      console.log(
        `${filePath} DONE!, remain promises: ${--fileLength}`
      );
      resolve();
      // analyzLog(threadIndex);
    }));
  });
  return Promise.all(promisePool);
};

(async () => {
  const start=Date.now();
  console.log(`Start statistic algorithm, timestamp: ${new Date(start).toJSON()}`);
  await statistic();
  const end=Date.now();
  console.log(`End statistic algorithm, timestamp: ${new Date(end).toJSON()}`);
  console.log(`It take ${(end-start)/1000} sec`);
})();

输出：

Start statistic algorithm, timestamp: 2023-05-29T10:50:42.550Z
.\log3.log DONE!, remain promises: 9
.\log2.log DONE!, remain promises: 8
.\log1.log DONE!, remain promises: 7
.\log4.log DONE!, remain promises: 6
.\log6.log DONE!, remain promises: 5
.\log5.log DONE!, remain promises: 4
.\log8.log DONE!, remain promises: 3
.\log7.log DONE!, remain promises: 2
.\log10.log DONE!, remain promises: 1
.\log9.log DONE!, remain promises: 0
End statistic algorithm, timestamp: 2023-05-29T10:50:42.586Z
It take 0.036 sec

赞(0）回复(0）举报 2023-06-05

我来回答

NodeJS 如何增强我的日志统计功能？

1条答案

相关问题

热门标签

最新问答