1）PaddlePaddle版本：1.5.2
2）CPU：请提供CPU型号，MKL/OpenBlas/MKLDNN/等数学库的使用情况
3）GPU：请提供GPU型号，CUDA10.0和CUDNN7.4
4）系统环境：centos6.3，Python3.6
import paddle.fluid as fluid
import os
os.sys.path.append('..')
import multiprocessing
import math
from functools import reduce
import argparse
from Models.Detectors import build_detector
from DataSets import COCOReader
from Optimizers import OPTIMIZERS
from Registry import build_from_cfg
from tools import get_logger,save_model
def build_optimizer(optimizer_cfg:dict):
return build_from_cfg(optimizer_cfg,OPTIMIZERS)
def get_input_tensors(img_info:dict):
img_size = img_info.get('Resize').get('expected_size') # w,h
size_divisor = img_info.get('Pad').get('size_divisor')
anchor_strides = img_info.get('anchor_strides')
featmap_sizes = list(map(lambda x: (img_size[1] // x, img_size[0] // x), anchor_strides))
num_elements = reduce(lambda x, y: x * y, list(map(lambda xy: xy[0]*xy[1], featmap_sizes)))
img_size = tuple(map(lambda x: math.ceil(x / size_divisor) * size_divisor, img_size))
train,val=[],[]
train.append(fluid.layers.data('img', [3, img_size[1], img_size[0]]))
val.append(fluid.layers.data('val_img', [3, img_size[1], img_size[0]]))
train.append(fluid.layers.data('bboxes', [4],lod_level=1))
train.append(fluid.layers.data('lod', [1],dtype='int32'))#*****************
val.append(fluid.layers.data('val_bboxes', [4],lod_level=1))
#val.append(fluid.layers.data('val_lod', [1]))#*******************8
train.append(fluid.layers.data('labels', [1],dtype='int32',lod_level=1))
val.append(fluid.layers.data('val_labels', [1],lod_level=1))
train.append(fluid.layers.data('batch_loc_target', [num_elements]))
train.append(fluid.layers.data('batch_loc_weight', [num_elements]))
train.append(fluid.layers.data('batch_loc_avg_factor', [1]))
train.append(fluid.layers.data('batch_multi_level_squares', [num_elements, 4]))
val.append(fluid.layers.data('val_batch_multi_level_squares', [num_elements, 4]))
train.append(fluid.layers.data('batch_bbox_anchors', [num_elements, 4]))
train.append(fluid.layers.data('batch_bbox_gts', [num_elements, 4]))
train.append(fluid.layers.data('batch_bbox_weights', [num_elements, 4]))
train.append(fluid.layers.data('batch_anchor_total_num', [1]))
return train,val
def build_program(main_prog:fluid.Program,
startup_prog:fluid.Program,
img_info:dict,
model,
optimizer_cfg:dict,
is_train:bool=True):
#train_fead_list, val_fead_list = get_input_tensors(img_info)
with fluid.program_guard(main_prog,startup_prog):
train_feed_list, val_feed_list = get_input_tensors(img_info)
PyReader=fluid.io.PyReader(feed_list=train_feed_list,
capacity=64,
use_double_buffer=True,
iterable=False,
return_list=False)
if is_train:
#return PyReader,train_feed_list[1]
with fluid.unique_name.guard('train'):
loss=model.forward_train(*train_feed_list)
optimizer=build_optimizer(optimizer_cfg)()
optimizer.minimize(loss,startup_program=None, parameter_list=None, no_grad_set=None, grad_clip=None)
return PyReader,loss
else:
with fluid.unique_name.guard('val'):
ValPyReader=fluid.io.PyReader(feed_list=val_feed_list,
capacity=64,
use_double_buffer=True,
iterable=True,
return_list=False)
map_evaluator=model.eval(*val_fead_list)
return ValPyReader,map_evaluator
def train(model,
config,
args,
logger,
train_reader,
val_reader=None):
start_prog=fluid.Program()
train_prog=fluid.Program()
val_prog=fluid.Program()
PyReader,loss=build_program(train_prog,start_prog,config.img_info,model,config.optimizer)
if val_reader is not None:
ValPyReader,map_evaluator=build_program(val_prog,start_prog,config.img_info,model,config.optimizer,False)
use_gpu=config.use_gpu
if not use_gpu:
devices_num = int(os.environ.get('CPU_NUM',
multiprocessing.cpu_count()))
place=fluid.CPUPlace()
places=[place]
else:
devices_num = fluid.core.get_cuda_device_count()
place=fluid.CUDAPlace(0)
places=fluid.cuda_places()
batch_size=config.data.get('imgs_per_gpu')
batch_size*=len(places)
PyReader.decorate_sample_generator(train_reader, batch_size, drop_last=True,places=places)
if val_reader is not None:
ValPyReader.decorate_sample_generator(val_reader,batch_size,drop_last=False,places=places)
exe=fluid.Executor(place)
exe.run(start_prog)
pretrained_model=False#args.pretrained_model
if pretrained_model:
def if_exist(var):
return os.path.exists(os.path.join(pretrained_model, var.name))
logger.info('load pretrained model:{}'.format(pretrained_model))
fluid.io.load_vars(exe, pretrained_model, main_program=train_prog,
predicate=if_exist)
if config.parallel:
compile_prog=fluid.CompiledProgram(train_prog).with_data_parallel(loss.name)
else:
compile_prog=train_prog
#exe.run(compile_prog)
def eval(epoch,best_map):
cur_map,accum_map=map_evaluator.get_map_var()
map_evaluator.reset(exe)
for data in ValPyReader():
cur,accum=exe.run(val_prog,feed=data,fetch_list=[cur_map.name,accum_map.name])
logger.info("{}:cur_map:{}|accum_map:{}".format(eval.__name__,cur,accum))
for epoch in range(config.total_epochs):
step=0
sum_loss=0
best_loss=100
PyReader.start()
while True:
try:
out=exe.run(compile_prog,fetch_list=[loss.name])
print(len(out),out)
logger.info('epoch:{}:step:{}|batch_size:{}|loss:{}|best loss:{}'.format(epoch,
step,
batch_size,
out[0],
best_loss))
except fluid.core.EOFException:
PyReader.reset()
break
"""
for data in PyReader():
# fluid.layers.Print(loss)
out=exe.run(compile_prog,feed=data,fetch_list=[loss.name])
# out=exe.run(compile_prog,feed=data,fetch_list=['bboxes'],return_numpy=False)
# print(loss)
# fluid.layers.Print(loss)
print(len(out),out)
logger.info('epoch:{}:step:{}|batch_size:{}|loss:{}|best loss:{}'.format(epoch,
step,
batch_size,
out[0],
best_loss))
step+=1"""
#if best_loss>out:
#save_model(config.work_dir,'best_model',train_prog,exe,logger)
#best_loss=out
#sum_loss+=out
#logger.info("epoch:{}|sum loss:{}".format(epoch,sum_loss))
if epoch%10==0:
if val_reader is not None:
eval()
else:
pass
if __name__=='__main__':
from importlib import import_module
parser=argparse.ArgumentParser()
parser.add_argument('--config',default='.ga_rpn_resnet50_fpn',type=str,help='train config file path')
parser.add_argument('--work_dir',default='./work_dirs/ga_rpn_r50_caffe_fpn_1x',type=str, help='the dir to save logs and models')
args=parser.parse_args()
config_file=args.config#'.ga_rpn_resnet50_fpn'
config=import_module(config_file,'configure')
model=build_detector(config.model,config.train_cfg,config.test_cfg)
train_reader=COCOReader(config.data.get('train'),rpn_cfg=config.train_cfg.get('rpn'),**config.model.get('rpn_head'))
#val_reader=COCOReader(config.data.get('val'),rpn_cfg=config.train_cfg.get('rpn'),**config.model.get('rpn_head'))
logger=get_logger()
logger.info("config.data:{}".format(config.data))
if config.use_gpu:
batch_size=len(config.gpu_nums.split(','))*config.data.get('imgs_per_gpu')
else:
batch_size=config.data.get('imgs_per_gpu')
os.environ['CUDA_VISIBLE_DEVICES'] = config.gpu_nums
train(model,config,args,logger,train_reader)报错信息
EnforceNotMet Traceback (most recent call last)
~/liubofang/jupyter/root/ga/tools/train.py in <module>
177 batch_size=config.data.get('imgs_per_gpu')
178 os.environ['CUDA_VISIBLE_DEVICES'] = config.gpu_nums
--> 179 train(model,config,args,logger,train_reader)
180 import numpy as np
181 m=fluid.Program()
~/liubofang/jupyter/root/ga/tools/train.py in train(model, config, args, logger, train_reader, val_reader)
78 train_prog=fluid.Program()
79 val_prog=fluid.Program()
---> 80 PyReader,loss=build_program(train_prog,start_prog,config.img_info,model,config.optimizer)
81 if val_reader is not None:
82 ValPyReader,map_evaluator=build_program(val_prog,start_prog,config.img_info,model,config.optimizer,False)
~/liubofang/jupyter/root/ga/tools/train.py in build_program(main_prog, startup_prog, img_info, model, optimizer_cfg, is_train)
52 use_double_buffer=True,
53 iterable=False,
---> 54 return_list=False)
55 if is_train:
56 #return PyReader,train_feed_list[1]
~/.conda/envs/paddle_10/lib/python3.6/site-packages/paddle/fluid/reader.py in __init__(self, feed_list, capacity, use_double_buffer, iterable, return_list)
209 self._capacity = capacity
210 if not self._iterable:
--> 211 self._init_non_iterable()
212
213 def _init_iterable(self, places):
~/.conda/envs/paddle_10/lib/python3.6/site-packages/paddle/fluid/reader.py in _init_non_iterable(self)
253 'shape_concat': shape_concat,
254 'lod_levels': lod_levels,
--> 255 'ranks': ranks
256 })
257
~/.conda/envs/paddle_10/lib/python3.6/site-packages/paddle/fluid/framework.py in append_op(self, *args,**kwargs)
1772 inputs=kwargs.get("inputs", None),
1773 outputs=kwargs.get("outputs", None),
-> 1774 attrs=kwargs.get("attrs", None))
1775
1776 self.ops.append(op)
~/.conda/envs/paddle_10/lib/python3.6/site-packages/paddle/fluid/framework.py in __init__(***failed resolving arguments***)
1162 self._update_desc_attr(attr_name, attr_val)
1163
-> 1164 self.desc.check_attrs()
1165 if self._has_kernel(type):
1166 self.desc.infer_var_type(self.block.desc)
EnforceNotMet: Cannot get attribute shape_concat by type std::vector<int, std::allocator<int> >, its type is std::vector<long, std::allocator<long> > at [/paddle/paddle/fluid/framework/attribute.h:42]
PaddlePaddle Call Stacks:
0 0x7f5b7153e278p void paddle::platform::EnforceNotMet::Init<std::string>(std::string, char const*, int) + 360
1 0x7f5b7153e5c7p paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int) + 87
2 0x7f5b717857b2p paddle::framework::ExtractAttribute<std::vector<int, std::allocator<int> > >::operator()(boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>&) const + 354
3 0x7f5b7178c550p std::_Function_handler<void (std::unordered_map<std::string, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > >*), paddle::framework::TypedAttrChecker<std::vector<int, std::allocator<int> > > >::_M_invoke(std::_Any_data const&, std::unordered_map<std::string, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, boost::variant<boost::blank, int, float, std::string, std::vector<int, std::allocator<int> >, std::vector<float, std::allocator<float> >, std::vector<std::string, std::allocator<std::string> >, bool, std::vector<bool, std::allocator<bool> >, paddle::framework::BlockDesc*, long, std::vector<paddle::framework::BlockDesc*, std::allocator<paddle::framework::BlockDesc*> >, std::vector<long, std::allocator<long> >, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> > > >*) + 272
4 0x7f5b716ae7d0p paddle::framework::OpDesc::CheckAttrs() + 304
5 0x7f5b715ea79fp
6 0x7f5b71570d56p
7 0x7f5bc0d52c54p _PyCFunction_FastCallDict + 340
8 0x7f5bc0ddac0ep
9 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
10 0x7f5bc0dd3e66p
11 0x7f5bc0dd5598p _PyFunction_FastCallDict + 984
12 0x7f5bc0d5301fp _PyObject_FastCallDict + 623
13 0x7f5bc0d57aa3p _PyObject_Call_Prepend + 99
14 0x7f5bc0d52a5ep PyObject_Call + 62
15 0x7f5bc0dab9ebp
16 0x7f5bc0ddadf7p
17 0x7f5bc0d52e3bp _PyObject_FastCallDict + 139
18 0x7f5bc0dd518ap _PyObject_FastCallKeywords + 170
19 0x7f5bc0ddac0ep
20 0x7f5bc0dfe51cp _PyEval_EvalFrameDefault + 4300
21 0x7f5bc0dd3e66p
22 0x7f5bc0dd4ed6p
23 0x7f5bc0ddab95p
24 0x7f5bc0dfe51cp _PyEval_EvalFrameDefault + 4300
25 0x7f5bc0dd4c5bp
26 0x7f5bc0ddab95p
27 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
28 0x7f5bc0dd3e66p
29 0x7f5bc0dd5598p _PyFunction_FastCallDict + 984
30 0x7f5bc0d5301fp _PyObject_FastCallDict + 623
31 0x7f5bc0d57aa3p _PyObject_Call_Prepend + 99
32 0x7f5bc0d52a5ep PyObject_Call + 62
33 0x7f5bc0dab9ebp
34 0x7f5bc0ddadf7p
35 0x7f5bc0d52e3bp _PyObject_FastCallDict + 139
36 0x7f5bc0dd518ap _PyObject_FastCallKeywords + 170
37 0x7f5bc0ddac0ep
38 0x7f5bc0dfe51cp _PyEval_EvalFrameDefault + 4300
39 0x7f5bc0dd3e66p
40 0x7f5bc0dd4e73p
41 0x7f5bc0ddab95p
42 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
43 0x7f5bc0dd429ep
44 0x7f5bc0dd4ed6p
45 0x7f5bc0ddab95p
46 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
47 0x7f5bc0dd59b9p PyEval_EvalCodeEx + 809
48 0x7f5bc0dd675cp PyEval_EvalCode + 28
49 0x7f5bc0dfb167p
50 0x7f5bc0d52b91p _PyCFunction_FastCallDict + 145
51 0x7f5bc0ddaabcp
52 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
53 0x7f5bc0dd3e66p
54 0x7f5bc0dd4e73p
55 0x7f5bc0ddab95p
56 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
57 0x7f5bc0dd4401p
58 0x7f5bc0dd4ed6p
59 0x7f5bc0ddab95p
60 0x7f5bc0dfe51cp _PyEval_EvalFrameDefault + 4300
61 0x7f5bc0dd429ep
62 0x7f5bc0dd4ed6p
63 0x7f5bc0ddab95p
64 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
65 0x7f5bc0dd5ff6p PyEval_EvalCodeEx + 2406
66 0x7f5bc0dd68e6p
67 0x7f5bc0d52a5ep PyObject_Call + 62
68 0x7f5bc0dfee37p _PyEval_EvalFrameDefault + 6631
69 0x7f5bc0dd3e66p
70 0x7f5bc0dd4ed6p
71 0x7f5bc0ddab95p
72 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
73 0x7f5bc0dd3e66p
74 0x7f5bc0dd5598p _PyFunction_FastCallDict + 984
75 0x7f5bc0d5301fp _PyObject_FastCallDict + 623
76 0x7f5bc0d57aa3p _PyObject_Call_Prepend + 99
77 0x7f5bc0d52a5ep PyObject_Call + 62
78 0x7f5bc0dfee37p _PyEval_EvalFrameDefault + 6631
79 0x7f5bc0dd3e66p
80 0x7f5bc0dd4e73p
81 0x7f5bc0ddab95p
82 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
83 0x7f5bc0dd59b9p PyEval_EvalCodeEx + 809
84 0x7f5bc0dd675cp PyEval_EvalCode + 28
85 0x7f5bc0dfb167p
86 0x7f5bc0d52b91p _PyCFunction_FastCallDict + 145
87 0x7f5bc0ddaabcp
88 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
89 0x7f5bc0dddbe6p _PyGen_Send + 598
90 0x7f5bc0dfe89fp _PyEval_EvalFrameDefault + 5199
91 0x7f5bc0dddbe6p _PyGen_Send + 598
92 0x7f5bc0dfe89fp _PyEval_EvalFrameDefault + 5199
93 0x7f5bc0dddbe6p _PyGen_Send + 598
94 0x7f5bc0d52c15p _PyCFunction_FastCallDict + 277
95 0x7f5bc0ddaabcp
96 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
97 0x7f5bc0dd4c5bp
98 0x7f5bc0ddab95p
99 0x7f5bc0dfd75ap _PyEval_EvalFrameDefault + 778
4条答案
按热度按时间z18hc3ub1#
你好，能否在这一行
print(shape_concat)发出来看看？46qrfjad2#
打印出来是这样的
rpppsulh3#
应该是因为数据shape太大了，目前PyReader不支持这么大的shape：1128666204000000000
pieyvz9o4#
嗯，好吧