python 将gzip压缩的Json对象写入Json文件而不加载它

2fjabf4q  于 2023-02-07  发布在  Python
关注(0)|答案(1)|浏览(179)

我想把dict作为gzip压缩的json对象写入一个json文件。
我有一些解决方案,但随着文件变大,附加过程变得越来越慢。所以加载文件不是办法。
我在这里找到了解决方案:

def append_record_seek(data,filename):
    print('append_record_seek started with data:{data} filename:{filename}')
    with open (filename, mode="r+") as file:
        file.seek(os.stat(filename).st_size -1)
        file.write( ",]".format(json.dumps(data)) )

稍后,我想读取该文件作为一个字典列表。
下面是我的最小代码示例:

import global_variables as gv
import time
import json as json
import base64
import io
import sys
import cv2
import gzip

import numpy as np

import os

from numpy import asarray
from json import JSONEncoder



data = {
  "brand": "Ford",
  "model": "Mustang",
  "year": 1964
}
path = r'C:/Videotool/Data'
name = 'test'
filename = path + '/' + name + '.json'
isExist = os.path.exists(path)
if not isExist:
    os.makedirs(path)
os.chdir(path)



def first_writer(data,filename):
    print(f'first_writer started with data:{data} filename:{filename}')
    with open (filename, 'w') as file:
        file.write('[')
        file.write(json.dumps(data))
        file.write(',')
        file.write(']')

        
def append_record_seek(data,filename):
    print('append_record_seek started with data:{data} filename:{filename}')
    with open (filename, mode="r+") as file:
        file.seek(os.stat(filename).st_size -1)
        file.write( ",]".format(json.dumps(data)) )



for x in range(6):
    print(f'step:{x}')
    file_exists = os.path.exists(name+'.json')
    if file_exists:
        print('file_exists')
        append_record_seek(data,filename)
    else:
        print('else')
        first_writer(data,filename)

未压缩的结果应如下所示:

[{"brand": "Ford", "model": "Mustang", "year": 1964},
{"brand": "Ford", "model": "Mustang", "year": 1964},
{"brand": "Ford", "model": "Mustang", "year": 1964},
{"brand": "Ford", "model": "Mustang", "year": 1964},
{"brand": "Ford", "model": "Mustang", "year": 1964}]

我的结果是:[{"brand": "Ford", "model": "Mustang", "year": 1964},,,,,,]
如果那有用的话,我想在写作前把垃圾压缩一下。
我希望有人能帮忙
更新:我已经得到了正确的Json格式:

def first_writer(data,filename):
    print(f'first_writer started with data:{data} filename:{filename}')
    with open (filename, 'w') as file:
        file.write( "[{}]".format(json.dumps(data)) )

        
def append_record_seek(data,filename):
    print('append_record_seek started with data:{data} filename:{filename}')
    with open (filename, mode="r+") as file:
        file.seek(os.stat(filename).st_size -1)
        file.write( ",{}]".format(json.dumps(data)) )

现在我得把拉链拉上...

7uhlpewt

7uhlpewt1#

注意:这不是问题的答案,因为没有,这只是强调可以生成一个压缩文件并在稍后解压缩 *,但 * 它不是有效的json。

import gzip
from copy import copy
import json

# just test data
x = {
  "brand": "Ford",
  "model": "Mustang",
  "year": 1964
}
z = {
  "brand": "Mato",
  "model": "Laatikko",
  "year": 2023
}

l = []

# populate the initial "json" in the list l
for i in range(3):
  y = copy(x)
  y["year"] += i
  l.append(y)

# write list of dicts as jsons string into file and compress it via gzip
# it doesnt really matter how this was originally done..
with open("data.gz", "wb") as f:
   f.write(gzip.compress(bytes(json.dumps(l, indent=2),"utf-8")))

# then, append a new entry to the same file -- which will get uncompressed
# with the previously stored *valid* json structure..

with open("data.gz", "ab") as f:
   f.write(gzip.compress(bytes(json.dumps(z, indent=2),"utf-8")))

这将导致文件在解压缩时如下所示

[
  {
    "brand": "Ford",
    "model": "Mustang",
    "year": 1964
  },
  {
    "brand": "Ford",
    "model": "Mustang",
    "year": 1965
  },
  {
    "brand": "Ford",
    "model": "Mustang",
    "year": 1966
  }
]{
  "brand": "Mato",
  "model": "Laatikko",
  "year": 2023
}

相关问题