使用fs在nodejs中阅读大型json文件

我有一个几乎500 MB的文件。这是从Firebase导出的。现在我想把这些数据添加到MySQL中。这是JSON文件的格式。

{
    "103597192838847432678": {
        "Galaxy A213": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A231": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A233": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        }       
    },
    "103597192838847432679": {
        "Galaxy A213": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A231": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A233": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        }       
    },
   "103597192838847432690": {
        "Galaxy A213": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A231": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A233": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        }       
    }
}

尝试的是这个

const express = require("express");
const db = require("../config/db");
const fs = require('fs');
const JSONStream = require('JSONStream');
const filePath = "./src/data/data.json";
const router = express.Router();
router.post("/", checkServer);
router.get("/", checkServer);
router.put("/", checkServer);
router.delete("/", checkServer);
module.exports = router;

async function checkServer(req, res, next) {
    const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' } );
    //const parser = JSONStream.parse('*');
    const parser = JSONStream.parse('*.*');
    fileStream.pipe(parser);
    
    parser.on('data', (jsonObject) => {
        console.log(jsonObject);
    });
      
    parser.on('end', () => {
        console.log('All JSON objects processed.');
    });
      
    res.json({
        status: true,
        successMessage: "Server is running Here",
    });
}

我在jsonObject中得到的数据是

"Galaxy A231": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        }

它缺少对象的键。我怎么能像这样读取整个对象呢？

"103597192838847432678": {
        "Galaxy A213": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A231": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        },
        "Galaxy A233": {
           "battery_level": "61",
           "connected_Wifi": "Mobile Data",
           "email_id": "canilda28@gmail",
           "last_Location_Lat": "11.8397285",
           "last_Location_Lng": "-15.6554222",
           "last_seen": "1697471866857",
           "tokenKey":"dL2bf4bnRNGgEn"
        }       
    }

只需传递emitKey: true参数，就会得到key和value对对象
更多关于DOC

async function checkServer(req, res, next) {
    const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' } );

    const parser = fileStream.pipe(JSONStream.parse([{emitKey: true}])); //here
    
    parser.on('data', async (jsonObject) => {
        parser.pause(); // this will pause the next read
        const key = jsonObject.key;
        const value = jsonObject.value;
        await somefunction(key,value); // insert into db
        console.log(key,value);
        parser.resume(); // continue to read next object
    });
      
    parser.on('end', () => {
        console.log('All JSON objects processed.');
    });
      
    res.json({
        status: true,
        successMessage: "Server is running Here",
    });
}

使用fs在nodejs中阅读大型json文件

1条答案

相关问题

热门标签

最新问答