模型测试时报错:
terminate called after throwing an instance of 'paddle::platform::EnforceNotMet' what(): an illegal memory access was encountered at [/paddle/newop/Paddle/paddle/fluid/platform/device_context.cc:102] PaddlePaddle Call Stacks: 0 0x7feb1b158950p void paddle::platform::EnforceNotMet::Init<char const*>(char const*, char const*, int) + 352 1 0x7feb1b158cc9p paddle::platform::EnforceNotMet::EnforceNotMet(std::__exception_ptr::exception_ptr, char const*, int) + 137 2 0x7feb1d09fb52p 3 0x7feb1d0abb84p paddle::platform::TemporaryAllocator::Release(std::function<void ()> const&) + 100 4 0x7feb1d0ac0fbp paddle::platform::TemporaryAllocator::FreeImpl(paddle::memory::allocation::Allocation*) + 747 5 0x7feb1ccfa554p paddle::operators::math::ConcatFunctor<paddle::platform::CUDADeviceContext, int>::operator()(paddle::platform::CUDADeviceContext const&, std::vector<paddle::framework::Tensor, std::allocator<paddle::framework::Tensor> > const&, int, paddle::framework::Tensor*) + 1044 6 0x7feb1b92663dp paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, int>::Compute(paddle::framework::ExecutionContext const&) const + 829 7 0x7feb1b926813p std::_Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 3ul, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, long>, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, int> >::operator()(char const*, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::_M_invoke(std::_Any_data const&, paddle::framework::ExecutionContext const&) + 35 8 0x7feb1d02bff7p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 375 9 0x7feb1d02c3d1p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529 10 0x7feb1d029b4cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332 11 0x7feb1b2e0dfep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 606 12 0x7feb1b2e3a1fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocator<std::string> > const&, bool) + 143 13 0x7feb1b149ffdp 14 0x7feb1b18b306p 15 0x7febc813585bp _PyMethodDef_RawFastCallKeywords + 795 16 0x7febc81358d5p _PyCFunction_FastCallKeywords + 37 17 0x7febc810dcc8p _PyEval_EvalFrameDefault + 26136 18 0x7febc8220c0bp _PyEval_EvalCodeWithName + 2619 19 0x7febc8135328p _PyFunction_FastCallKeywords + 168 20 0x7febc810f5a4p _PyEval_EvalFrameDefault + 32500 21 0x7febc8220c0bp _PyEval_EvalCodeWithName + 2619 22 0x7febc8135328p _PyFunction_FastCallKeywords + 168 23 0x7febc810f5a4p _PyEval_EvalFrameDefault + 32500 24 0x7febc8106500p 25 0x7febc810faf0p _PyEval_EvalFrameDefault + 33856 26 0x7febc8220c0bp _PyEval_EvalCodeWithName + 2619 27 0x7febc8220d4dp PyEval_EvalCodeEx + 109 28 0x7febc8220d9bp PyEval_EvalCode + 59 29 0x7febc825bee2p PyRun_FileExFlags + 178 30 0x7febc825c047p PyRun_SimpleFileExFlags + 231 31 0x7febc827c9d2p 32 0x7febc827d25cp _Py_UnixMain + 108 33 0x7febc71c6445p __libc_start_main + 245 34 0x4006c1p
3条答案
按热度按时间2skhul331#
terminate called after throwing an instance of 'paddle::platform::EnforceNotMet'
what(): an illegal memory access was encountered at [/paddle/newop/Paddle/paddle/fluid/platform/device_context.cc:102]
PaddlePaddle Call Stacks:
0 0x7feb1b158950p void paddle::platform::EnforceNotMet::Init<char const*>(char const*, char const*, int) + 352
1 0x7feb1b158cc9p paddle::platform::EnforceNotMet::EnforceNotMet(std::exception_ptr::exception_ptr, char const, int) + 137
2 0x7feb1d09fb52p
3 0x7feb1d0abb84p paddle::platform::TemporaryAllocator::Release(std::function<void ()> const&) + 100
4 0x7feb1d0ac0fbp paddle::platform::TemporaryAllocator::FreeImpl(paddle::memory::allocation::Allocation) + 747
5 0x7feb1ccfa554p paddle::operators::math::ConcatFunctor<paddle::platform::CUDADeviceContext, int>::operator()(paddle::platform::CUDADeviceContext const&, std::vector<paddle::framework::Tensor, std::allocatorpaddle::framework::Tensor > const&, int, paddle::framework::Tensor*) + 1044
6 0x7feb1b92663dp paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, int>::Compute(paddle::framework::ExecutionContext const&) const + 829
7 0x7feb1b926813p std::Function_handler<void (paddle::framework::ExecutionContext const&), paddle::framework::OpKernelRegistrarFunctor<paddle::platform::CUDAPlace, false, 3ul, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, double>, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, float>, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, long>, paddle::operators::ConcatKernel<paddle::platform::CUDADeviceContext, int> >::operator()(char const, char const*, int) const::{lambda(paddle::framework::ExecutionContext const&)#1}>::*M_invoke(std::Any_data const&, paddle::framework::ExecutionContext const&) + 35
8 0x7feb1d02bff7p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void, boost::detail::variant::void*, boost::detail::variant::void*, boost::detail::variant::void*, boost::detail::variant::void*, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&, paddle::framework::RuntimeContext*) const + 375
9 0x7feb1d02c3d1p paddle::framework::OperatorWithKernel::RunImpl(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) const + 529
10 0x7feb1d029b4cp paddle::framework::OperatorBase::Run(paddle::framework::Scope const&, boost::variant<paddle::platform::CUDAPlace, paddle::platform::CPUPlace, paddle::platform::CUDAPinnedPlace, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> const&) + 332
11 0x7feb1b2e0dfep paddle::framework::Executor::RunPreparedContext(paddle::framework::ExecutorPrepareContext*, paddle::framework::Scope*, bool, bool, bool) + 606
12 0x7feb1b2e3a1fp paddle::framework::Executor::Run(paddle::framework::ProgramDesc const&, paddle::framework::Scope*, int, bool, bool, std::vector<std::string, std::allocatorstd::string > const&, bool) + 143
13 0x7feb1b149ffdp
14 0x7feb1b18b306p
15 0x7febc813585bp _PyMethodDef_RawFastCallKeywords + 795
16 0x7febc81358d5p _PyCFunction_FastCallKeywords + 37
17 0x7febc810dcc8p _PyEval_EvalFrameDefault + 26136
18 0x7febc8220c0bp _PyEval_EvalCodeWithName + 2619
19 0x7febc8135328p _PyFunction_FastCallKeywords + 168
20 0x7febc810f5a4p _PyEval_EvalFrameDefault + 32500
21 0x7febc8220c0bp _PyEval_EvalCodeWithName + 2619
22 0x7febc8135328p _PyFunction_FastCallKeywords + 168
23 0x7febc810f5a4p _PyEval_EvalFrameDefault + 32500
24 0x7febc8106500p
25 0x7febc810faf0p _PyEval_EvalFrameDefault + 33856
26 0x7febc8220c0bp _PyEval_EvalCodeWithName + 2619
27 0x7febc8220d4dp PyEval_EvalCodeEx + 109
28 0x7febc8220d9bp PyEval_EvalCode + 59
29 0x7febc825bee2p PyRun_FileExFlags + 178
30 0x7febc825c047p PyRun_SimpleFileExFlags + 231
31 0x7febc827c9d2p
32 0x7febc827d25cp _Py_UnixMain + 108
33 0x7febc71c6445p __libc_start_main + 245
34 0x4006c1p
pjngdqdw2#
请问是必现的问题还是偶发的,是一开始就挂了,还是跑了几个迭代后的
drkbr07n3#
稳定出现,一开始就挂了