pytorch UnpicklingError:遇到加载持久ID指令,但未指定persistent_load函数

yyyllmsg  于 12个月前  发布在  其他
关注(0)|答案(1)|浏览(243)

我试图运行一个名为api.py的python文件。在这个文件中,我加载了使用PyTorch构建和训练的深度学习模型的pickle文件。
api.pyapi.py中,下面给出的函数是最重要的函数。

def load_model_weights(model_architecture, weights_path):
  if os.path.isfile(weights_path):
      cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
      model_architecture.load_state_dict(torch.load(weights_path))
  else:
      raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

字符串
目录结构

📦MP1
 ┣ 📂.ipynb_checkpoints
 ┃ ┗ 📜RS_netflix3months_100epochs_64,128,128-checkpoint.ipynb
 ┣ 📂data
 ┃ ┣ 📜AutoEncoder.png
 ┃ ┣ 📜collaborative_filtering.gif
 ┃ ┣ 📜movie_titles.txt
 ┃ ┗ 📜shut_up.gif
 ┣ 📂DeepRecommender
 ┃ ┣ 📂data_utils
 ┃ ┃ ┣ 📜movielens_data_convert.py
 ┃ ┃ ┗ 📜netflix_data_convert.py
 ┃ ┣ 📂reco_encoder
 ┃ ┃ ┣ 📂data
 ┃ ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┃ ┣ 📜input_layer.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┣ 📜input_layer_api.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ 📜input_layer.py
 ┃ ┃ ┃ ┣ 📜input_layer_api.py
 ┃ ┃ ┃ ┗ 📜__init__.py
 ┃ ┃ ┣ 📂model
 ┃ ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┃ ┣ 📜model.cpython-37.pyc
 ┃ ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┃ ┣ 📜model.py
 ┃ ┃ ┃ ┗ 📜__init__.py
 ┃ ┃ ┣ 📂__pycache__
 ┃ ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┃ ┗ 📜__init__.py
 ┃ ┣ 📂__pycache__
 ┃ ┃ ┗ 📜__init__.cpython-37.pyc
 ┃ ┣ 📜compute_RMSE.py
 ┃ ┣ 📜infer.py
 ┃ ┣ 📜run.py
 ┃ ┗ 📜__init__.py
 ┣ 📂model_save
 ┃ ┣ 📂model.epoch_99
 ┃ ┃ ┗ 📂archive
 ┃ ┃ ┃ ┣ 📂data
 ┃ ┃ ┃ ┃ ┣ 📜92901648
 ┃ ┃ ┃ ┃ ┣ 📜92901728
 ┃ ┃ ┃ ┃ ┣ 📜92901808
 ┃ ┃ ┃ ┃ ┣ 📜92901888
 ┃ ┃ ┃ ┃ ┣ 📜92901968
 ┃ ┃ ┃ ┃ ┣ 📜92902048
 ┃ ┃ ┃ ┃ ┣ 📜92902128
 ┃ ┃ ┃ ┃ ┣ 📜92902208
 ┃ ┃ ┃ ┃ ┣ 📜92902288
 ┃ ┃ ┃ ┃ ┣ 📜92902368
 ┃ ┃ ┃ ┃ ┣ 📜92902448
 ┃ ┃ ┃ ┃ ┗ 📜92902608
 ┃ ┃ ┃ ┣ 📜data.pkl
 ┃ ┃ ┃ ┗ 📜version
 ┃ ┣ 📜model.epoch_99.zip
 ┃ ┗ 📜model.onnx
 ┣ 📂Netflix
 ┃ ┣ 📂N1Y_TEST
 ┃ ┃ ┗ 📜n1y.test.txt
 ┃ ┣ 📂N1Y_TRAIN
 ┃ ┃ ┗ 📜n1y.train.txt
 ┃ ┣ 📂N1Y_VALID
 ┃ ┃ ┗ 📜n1y.valid.txt
 ┃ ┣ 📂N3M_TEST
 ┃ ┃ ┗ 📜n3m.test.txt
 ┃ ┣ 📂N3M_TRAIN
 ┃ ┃ ┗ 📜n3m.train.txt
 ┃ ┣ 📂N3M_VALID
 ┃ ┃ ┗ 📜n3m.valid.txt
 ┃ ┣ 📂N6M_TEST
 ┃ ┃ ┗ 📜n6m.test.txt
 ┃ ┣ 📂N6M_TRAIN
 ┃ ┃ ┗ 📜n6m.train.txt
 ┃ ┣ 📂N6M_VALID
 ┃ ┃ ┗ 📜n6m.valid.txt
 ┃ ┣ 📂NF_TEST
 ┃ ┃ ┗ 📜nf.test.txt
 ┃ ┣ 📂NF_TRAIN
 ┃ ┃ ┗ 📜nf.train.txt
 ┃ ┗ 📂NF_VALID
 ┃ ┃ ┗ 📜nf.valid.txt
 ┣ 📂test
 ┃ ┣ 📂testData_iRec
 ┃ ┃ ┣ 📜.part-00199-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt.crc
 ┃ ┃ ┣ 📜part-00000-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┣ 📜part-00003-f683aa3b-8840-4835-b8bc-a8d1eaa11c78.txt
 ┃ ┃ ┗ 📜_SUCCESS
 ┃ ┣ 📂testData_uRec
 ┃ ┃ ┣ 📜.part-00000-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt.crc
 ┃ ┃ ┣ 📜._SUCCESS.crc
 ┃ ┃ ┣ 📜part-00161-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┣ 📜part-00196-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┃ ┗ 📜part-00199-4a844096-8dd9-425e-9d9d-bd9062cc6940.txt
 ┃ ┣ 📜data_layer_tests.py
 ┃ ┣ 📜test_model.py
 ┃ ┗ 📜__init__.py
 ┣ 📂__pycache__
 ┃ ┣ 📜api.cpython-37.pyc
 ┃ ┣ 📜load_test.cpython-37.pyc
 ┃ ┣ 📜parameters.cpython-37.pyc
 ┃ ┗ 📜utils.cpython-37.pyc
 ┣ 📜api.py
 ┣ 📜compute_RMSE.py
 ┣ 📜load_test.py
 ┣ 📜logger.py
 ┣ 📜netflix_1y_test.csv
 ┣ 📜netflix_1y_train.csv
 ┣ 📜netflix_1y_valid.csv
 ┣ 📜netflix_3m_test.csv
 ┣ 📜netflix_3m_train.csv
 ┣ 📜netflix_3m_valid.csv
 ┣ 📜netflix_6m_test.csv
 ┣ 📜netflix_6m_train.csv
 ┣ 📜netflix_6m_valid.csv
 ┣ 📜netflix_full_test.csv
 ┣ 📜netflix_full_train.csv
 ┣ 📜netflix_full_valid.csv
 ┣ 📜parameters.py
 ┣ 📜preds.txt
 ┣ 📜RS_netflix3months_100epochs_64,128,128.ipynb
 ┗ 📜utils.py


我得到这样一个错误(serialization.py).有人可以帮助我这个错误?

D:\Anaconda\envs\practise\lib\site-packages\torch\serialization.py in _legacy_load(f, map_location, pickle_module, **pickle_load_args)
    762             "functionality.")
    763 
--> 764     magic_number = pickle_module.load(f, **pickle_load_args)
    765     if magic_number != MAGIC_NUMBER:
    766         raise RuntimeError("Invalid magic number; corrupt file?")

UnpicklingError: A load persistent id instruction was encountered,
but no persistent_load function was specified.

c3frrgcw

c3frrgcw1#

在搜索了PyTorch文档之后,我最终将模型保存为ONNX,然后将ONNX模型加载到PyTorch模型中并用于推理。

import onnx
from onnx2pytorch import ConvertModel

def load_model_weights(model_architecture, weights_path):
    if os.path.isfile("model.onnx"):
        cherrypy.log("CHERRYPYLOG Loading model from: {}".format(weights_path))
        onnx_model = onnx.load("model.onnx")
        pytorch_model = ConvertModel(onnx_model)
        ## model_architecture.load_state_dict(torch.load(weights_path))
    else:
        raise ValueError("Path not found {}".format(weights_path))

        
def load_recommender(vector_dim, hidden, activation, dropout, weights_path):

    rencoder_api = model.AutoEncoder(layer_sizes=[vector_dim] + [int(l) for l in hidden.split(',')],
                               nl_type=activation,
                               is_constrained=False,
                               dp_drop_prob=dropout,
                               last_layer_activations=False)
    load_model_weights(rencoder_api, weights_path) 
    rencoder_api.eval()
    rencoder_api = rencoder_api.cuda()
    return rencoder_api

字符串
一些有用的资源:
torch.save
torch.load
ONNX tutorials

相关问题