# -*- coding: utf-8 -*-
import paddle
import paddle.fluid as fluid
import json
import codecs
import numpy
import link_model
import time
import os
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
id2char, char2id = json.load(open('data_path/all_chars_me.json'))
print("Finish loaded char_dict!")
def get_train_data(filename):
def reader():
with codecs.open(filename, encoding="utf-8") as f:
for l in f:
_ = json.loads(l)
text = _['text']
kb_dict = _['kb_dict']
label = _['label']
new_slot = text + kb_dict
text_slot = [char2id.get(c, 1) for c in new_slot]
# kb_dict_slot = [char2id.get(c, 1) for c in kb_dict]
target_slot = numpy.zeros(2)
target_slot[int(label)] = 1
sample_result = [text_slot, target_slot]
if sample_result is None:
continue
yield tuple(sample_result)
print("Finish loaded train_data!")
return reader
def del_file(path):
ls = os.listdir(path)
for i in ls:
c_path = os.path.join(path, i)
if os.path.isdir(c_path):
del_file(c_path)
else:
os.remove(c_path)
char_len = len(char2id)
embedding_size = 64
batch_size = 128
def train():
# train_data = get_train_data("data_path/all_train_deal.json")
train_data = get_train_data("data_path/link_train_data.json")
dev_data = get_train_data("data_path/link_dev_data.json")
text = fluid.layers.data(name="text", shape=[1], dtype='int64', lod_level=1)
desc = fluid.layers.data(name="desc", shape=[1], dtype='int64', lod_level=1)
target = fluid.layers.data(
name='target', shape=[2], dtype='float32', lod_level=0)
feature_out = link_model.bilstm(text=text,desc=desc, embdding_size=embedding_size, char_len=char_len)
entropy_cost = fluid.layers.cross_entropy(input=feature_out, label=target, soft_label=True)
avg_cost = fluid.layers.mean(entropy_cost)
# valid program
valid_program = fluid.default_main_program().clone(for_test=True)
optimizer = fluid.optimizer.AdamOptimizer(learning_rate=0.001)
optimizer.minimize(avg_cost)
train_batch_reader = paddle.batch(
reader=paddle.reader.shuffle(train_data, buf_size=6144),
batch_size=batch_size
)
valid_batch_reader = paddle.batch(
reader=paddle.reader.shuffle(dev_data, buf_size=2048), batch_size=batch_size
)
# place = fluid.CPUPlace()
place = fluid.CUDAPlace(0)
feeder = fluid.DataFeeder(feed_list=[text,desc, target], place=place)
exe = fluid.Executor(place)
save_dirname = './output/link_model'
exe.run(fluid.default_startup_program())
start_time = time.time()
batch_id = 0
best_precision = 0.0
for pass_id in range(200):
pass_start_time = time.time()
cost_sum, cost_counter = 0, 0
for data in train_batch_reader():
# print(data[1][1])
cost = exe.run(program=fluid.default_main_program(), feed=feeder.feed(data),
fetch_list=[avg_cost])
# ss= numpy.array(feature_outs)
# sss= numpy.array(kb_dict_slot)
# print("feature_outs::::::")
# print(ss.shape)
# print(ss[1])
# print("targets::::::")
# print(sss.shape)
# print(sss[1])
# print("======================")
# cost = numpy.array(cost)
cost = cost[0]
cost_sum += cost
cost_counter += 1
if batch_id % 40 == 0 and batch_id != 0:
print >> sys.stderr, "batch %d finished, second per batch: %02f, avg_cost: %f" % (
batch_id, (time.time() - start_time) / batch_id, cost)
if batch_id % 200 == 0 and batch_id != 0:
all_target_count_0 = 0.0
all_target_count_1 = 0.0
all_correct_count_0 = 0.0
all_correct_count_1 = 0.0
for valid_data in valid_batch_reader():
targets, feature_outs = exe.run(program=valid_program, feed=feeder.feed(valid_data),
fetch_list=[target, feature_out],
return_numpy=False)
label_tag_scores = numpy.array(feature_outs)
target_label_tag_scores = numpy.array(targets)
for r in range(len(target_label_tag_scores)):
tag_pre = label_tag_scores[r].argmax()
tag_target = target_label_tag_scores[r].argmax()
if tag_target == 1:
all_target_count_1 += 1
if tag_target == tag_pre:
all_correct_count_1 += 1
else:
all_target_count_0 += 1
if tag_target == tag_pre:
all_correct_count_0 += 1
precision_1 = all_correct_count_1 / all_target_count_1
precision_0 = all_correct_count_0 / all_target_count_0
print(all_correct_count_1)
print(all_correct_count_0)
print(all_target_count_1)
print(all_target_count_0)
print('======================')
if precision_1 > best_precision:
save_path = os.path.join(save_dirname, 'best')
if os.path.exists(save_path) == False:
os.makedirs(save_path)
del_file(save_path)
with open(save_path + "/best_precision.txt", "w") as f:
f.write("precision_1:" + str(precision_1) + "\tprecision_0:" + str(precision_0))
fluid.io.save_inference_model(save_path, ['text','desc'],
[feature_out], exe, main_program=valid_program,
params_filename='params')
best_precision = precision_1
print >> sys.stderr, "Valid end, precision_1: %f,\tprecision_0: %f" % (precision_1, precision_0)
batch_id = batch_id + 1
pass_avg_cost = cost_sum / cost_counter if cost_counter > 0 else 0.0
print >> sys.stderr, "%d pass end, cost time: %02f, avg_cost: %f" % (
pass_id, time.time() - pass_start_time, pass_avg_cost)
else:
# pass times complete and the training is over
save_path = os.path.join(save_dirname, 'final')
fluid.io.save_inference_model(save_path, ['text','desc'],
[feature_out], exe, params_filename='params')
return
train()
6条答案
按热度按时间0x6upsns1#
我打印了一下global_block中的我那个变量的名字 是可以找到的,但是保存的时候就不行 paddlepaddle1.2 GPU
5q4ezhmt2#
代码里缩进是到哪儿,代码贴错了
d5vmydt93#
代码里缩进是到哪儿,代码贴错了
额 哪里的缩进?我是直接在pycharm里沾出来的
o2gm4chl4#
eaf3rand5#
link_model 里面是不是把output 定义在另外一个block里?
h43kikqp6#
link_model 里面是不是把output 定义在另外一个block里?
没有,我现在还没太学会用paddlepaddle的block,我只是按照对两个变量分别训练了词向量,然后再通过一个bilstm网络。模型最后能正常训练,但是一到保存的时候就会报那个错误