devbox
人工智能uu
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

语音识别——基于深度学习的中文语音识别系统框架

作者：人工智能uu | 2024-07-22 03:47:13

赞

踩

基于深度学习的中文语音识别系统

本语音识别程序分为声学模型部分和语言模型部分，框架分别采用keras和tensorflow。

程序中使用的完整开源数据集是thchs30、aishell、prime、stcmd，但在程序的演示中只使用了部分的thchs30的数据集。

下面代码是声学模型和语言模型的训练部分

  1. import os
  2. import tensorflow as tf
  3. from utils import get_data, data_hparams
  4. from keras.callbacks import ModelCheckpoint
  5.  
  6.  
  7. # 0.准备训练所需数据------------------------------
  8. data_args = data_hparams()
  9. data_args.data_type = 'train'
  10. data_args.data_path = 'data/'
  11. data_args.thchs30 = True
  12. data_args.aishell = True
  13. data_args.prime = True
  14. data_args.stcmd = True
  15. data_args.batch_size = 4
  16. data_args.data_length = 10
  17. # data_args.data_length = None
  18. data_args.shuffle = True
  19. train_data = get_data(data_args)
  20.  
  21. # 0.准备验证所需数据------------------------------
  22.  
  23. data_args = data_hparams()
  24. data_args.data_type = 'dev'
  25. # data_args.data_path = '../dataset/'
  26. data_args.data_path = 'data/'
  27. data_args.thchs30 = True
  28. data_args.aishell = True
  29. data_args.prime = True
  30. data_args.stcmd = True
  31. data_args.batch_size = 4
  32. # data_args.data_length = None
  33. data_args.data_length = 10
  34. data_args.shuffle = True
  35. dev_data = get_data(data_args)
  36.  
  37. # 1.声学模型训练-----------------------------------
  38. from model_speech.cnn_ctc import Am, am_hparams
  39. am_args = am_hparams()
  40. am_args.vocab_size = len(train_data.am_vocab)
  41. am_args.gpu_nums = 1
  42. am_args.lr = 0.0008
  43. am_args.is_training = True
  44. am = Am(am_args)
  45.  
  46. if os.path.exists('logs_am/model.h5'):
  47.     print('load acoustic model...')
  48.     am.ctc_model.load_weights('logs_am/model.h5')
  49.  
  50. epochs = 10
  51. batch_num = len(train_data.wav_lst) // train_data.batch_size
  52.  
  53. # checkpoint
  54. ckpt = "model_{epoch:02d}-{val_acc:.2f}.hdf5"
  55. checkpoint = ModelCheckpoint(os.path.join('./checkpoint', ckpt), monitor='val_loss', save_weights_only=False, verbose=1, save_best_only=True)
  56.  
  57.  
  58. batch = train_data.get_am_batch()
  59.  
  60.  
  61. am.ctc_model.fit_generator(batch, steps_per_epoch=batch_num, epochs=200, workers=1, use_multiprocessing=False)
  62.  
  63. am.ctc_model.save_weights('logs_am/model.h5')
  64.  
  65.  
  66. # 2.语言模型训练-------------------------------------------
  67. from model_language.transformer import Lm, lm_hparams
  68. lm_args = lm_hparams()
  69. lm_args.num_heads = 8
  70. lm_args.num_blocks = 6
  71. lm_args.input_vocab_size = len(train_data.pny_vocab)
  72. lm_args.label_vocab_size = len(train_data.han_vocab)
  73. lm_args.max_length = 100
  74. lm_args.hidden_units = 512
  75. lm_args.dropout_rate = 0.2
  76. lm_args.lr = 0.0003
  77. lm_args.is_training = True
  78. lm = Lm(lm_args)
  79.  
  80. epochs = 100
  81. with lm.graph.as_default():
  82.     saver =tf.train.Saver()
  83. with tf.Session(graph=lm.graph) as sess:
  84.     merged = tf.summary.merge_all()
  85.     sess.run(tf.global_variables_initializer())
  86.     add_num = 0
  87.     if os.path.exists('logs_lm/checkpoint'):
  88.         print('loading language model...')
  89.         latest = tf.train.latest_checkpoint('logs_lm')
  90.         add_num = int(latest.split('_')[-1])
  91.         saver.restore(sess, latest)
  92.     writer = tf.summary.FileWriter('logs_lm/tensorboard', tf.get_default_graph())
  93.     for k in range(epochs):
  94.         total_loss = 0
  95.         batch = train_data.get_lm_batch()
  96.         for i in range(batch_num):
  97.             input_batch, label_batch = next(batch)
  98.             feed = {lm.x: input_batch, lm.y: label_batch}
  99.             cost,_ = sess.run([lm.mean_loss,lm.train_op], feed_dict=feed)
  100.             total_loss += cost
  101.             if (k * batch_num + i) % 10 == 0:
  102.                 rs=sess.run(merged, feed_dict=feed)
  103.                 writer.add_summary(rs, k * batch_num + i)
  104.         print('epochs', k+1, ': average loss = ', total_loss/batch_num)
  105.     saver.save(sess, 'logs_lm/model_%d' % (epochs + add_num))
  106.     writer.close()

数据处理

  1. import difflib
  2. import numpy as np
  3. import tensorflow as tf
  4. import scipy.io.wavfile as wav
  5. from tqdm import tqdm
  6. from scipy.fftpack import fft
  7. from python_speech_features import mfcc
  8. from random import shuffle
  9. from keras import backend as K
  10.  
  11. def data_hparams():
  12.     params = tf.contrib.training.HParams(
  13.         # vocab
  14.         data_type='train',
  15.         data_path='data/',
  16.         thchs30=True,
  17.         aishell=True,
  18.         prime=True,
  19.         stcmd=True,
  20.         batch_size=1,
  21.         data_length=10,
  22.         shuffle=True)
  23.     return params
  24.  
  25.  
  26. class get_data():
  27.     def __init__(self, args):
  28.         self.data_type = args.data_type
  29.         self.data_path = args.data_path
  30.         self.thchs30 = args.thchs30
  31.         self.aishell = args.aishell
  32.         self.prime = args.prime
  33.         self.stcmd = args.stcmd
  34.         self.data_length = args.data_length
  35.         self.batch_size = args.batch_size
  36.         self.shuffle = args.shuffle
  37.         self.source_init()
  38.  
  39.     def source_init(self):
  40.         print('get source list...')
  41.         read_files = []
  42.         if self.data_type == 'train':
  43.             if self.thchs30 == True:
  44.                 read_files.append('thchs_train.txt')
  45.             if self.aishell == True:
  46.                 read_files.append('aishell_train.txt')
  47.             if self.prime == True:
  48.                 read_files.append('prime.txt')
  49.             if self.stcmd == True:
  50.                 read_files.append('stcmd.txt')
  51.         elif self.data_type == 'dev':
  52.             if self.thchs30 == True:
  53.                 read_files.append('thchs_dev.txt')
  54.             if self.aishell == True:
  55.                 read_files.append('aishell_dev.txt')
  56.         elif self.data_type == 'test':
  57.             if self.thchs30 == True:
  58.                 read_files.append('thchs_test.txt')
  59.             if self.aishell == True:
  60.                 read_files.append('aishell_test.txt')
  61.         self.wav_lst = []
  62.         self.pny_lst = []
  63.         self.han_lst = []
  64.         for file in read_files:
  65.             print('load ', file, ' data...')
  66.             sub_file = 'data/' + file
  67.             with open(sub_file, 'r', encoding='utf-8-sig') as f:
  68.                 data = f.readlines()
  69.             for line in tqdm(data):
  70.  
  71.                 wav_file, pny, han = line.split('\t')
  72.                 self.wav_lst.append(wav_file)
  73.                 self.pny_lst.append(pny.split(' '))
  74.                 self.han_lst.append(han.strip('\n'))
  75.         if self.data_length:
  76.             self.wav_lst = self.wav_lst[:self.data_length]
  77.             self.pny_lst = self.pny_lst[:self.data_length]
  78.             self.han_lst = self.han_lst[:self.data_length]
  79.         print('make am vocab...')
  80.         self.am_vocab = self.mk_am_vocab(self.pny_lst)
  81.         print('make lm pinyin vocab...')
  82.         self.pny_vocab = self.mk_lm_pny_vocab(self.pny_lst)
  83.         print('make lm hanzi vocab...')
  84.         self.han_vocab = self.mk_lm_han_vocab(self.han_lst)
  85.  
  86.     def get_am_batch(self):
  87.         shuffle_list = [i for i in range(len(self.wav_lst))]
  88.         while 1:
  89.             if self.shuffle == True:
  90.                 shuffle(shuffle_list)
  91.             for i in range(len(self.wav_lst) // self.batch_size):
  92.                 wav_data_lst = []
  93.                 label_data_lst = []
  94.                 begin = i * self.batch_size
  95.                 end = begin + self.batch_size
  96.                 sub_list = shuffle_list[begin:end]
  97.                 for index in sub_list:
  98.                     fbank = compute_fbank(self.data_path + self.wav_lst[index])
  99.                     pad_fbank = np.zeros((fbank.shape[0] // 8 * 8 + 8, fbank.shape[1]))
  100.                     pad_fbank[:fbank.shape[0], :] = fbank
  101.                     label = self.pny2id(self.pny_lst[index], self.am_vocab)
  102.                     label_ctc_len = self.ctc_len(label)
  103.                     if pad_fbank.shape[0] // 8 >= label_ctc_len:
  104.                         wav_data_lst.append(pad_fbank)
  105.                         label_data_lst.append(label)
  106.                 pad_wav_data, input_length = self.wav_padding(wav_data_lst)
  107.                 pad_label_data, label_length = self.label_padding(label_data_lst)
  108.                 inputs = {'the_inputs': pad_wav_data,
  109.                           'the_labels': pad_label_data,
  110.                           'input_length': input_length,
  111.                           'label_length': label_length,
  112.                           }
  113.                 outputs = {'ctc': np.zeros(pad_wav_data.shape[0], )}
  114.                 yield inputs, outputs
  115.  
  116.     def get_lm_batch(self):
  117.         batch_num = len(self.pny_lst) // self.batch_size
  118.         for k in range(batch_num):
  119.             begin = k * self.batch_size
  120.             end = begin + self.batch_size
  121.             input_batch = self.pny_lst[begin:end]
  122.             label_batch = self.han_lst[begin:end]
  123.             max_len = max([len(line) for line in input_batch])
  124.             input_batch = np.array(
  125.                 [self.pny2id(line, self.pny_vocab) + [0] * (max_len - len(line)) for line in input_batch])
  126.             label_batch = np.array(
  127.                 [self.han2id(line, self.han_vocab) + [0] * (max_len - len(line)) for line in label_batch])
  128.             yield input_batch, label_batch
  129.  
  130.     def pny2id(self, line, vocab):
  131.         return [vocab.index(pny) for pny in line]
  132.  
  133.     def han2id(self, line, vocab):
  134.         return [vocab.index(han) for han in line]
  135.  
  136.     def wav_padding(self, wav_data_lst):
  137.         wav_lens = [len(data) for data in wav_data_lst]
  138.         wav_max_len = max(wav_lens)
  139.         wav_lens = np.array([leng // 8 for leng in wav_lens])
  140.         new_wav_data_lst = np.zeros((len(wav_data_lst), wav_max_len, 200, 1))
  141.         for i in range(len(wav_data_lst)):
  142.             new_wav_data_lst[i, :wav_data_lst[i].shape[0], :, 0] = wav_data_lst[i]
  143.         return new_wav_data_lst, wav_lens
  144.  
  145.     def label_padding(self, label_data_lst):
  146.         label_lens = np.array([len(label) for label in label_data_lst])
  147.         max_label_len = max(label_lens)
  148.         new_label_data_lst = np.zeros((len(label_data_lst), max_label_len))
  149.         for i in range(len(label_data_lst)):
  150.             new_label_data_lst[i][:len(label_data_lst[i])] = label_data_lst[i]
  151.         return new_label_data_lst, label_lens
  152.  
  153.     def mk_am_vocab(self, data):
  154.         vocab = []
  155.         for line in tqdm(data):
  156.             line = line
  157.             for pny in line:
  158.                 if pny not in vocab:
  159.                     vocab.append(pny)
  160.         vocab.append('_')
  161.         return vocab
  162.  
  163.     def mk_lm_pny_vocab(self, data):
  164.         vocab = ['<PAD>']
  165.         for line in tqdm(data):
  166.             for pny in line:
  167.                 if pny not in vocab:
  168.                     vocab.append(pny)
  169.         return vocab
  170.  
  171.     def mk_lm_han_vocab(self, data):
  172.         vocab = ['<PAD>']
  173.         for line in tqdm(data):
  174.             line = ''.join(line.split(' '))
  175.             for han in line:
  176.                 if han not in vocab:
  177.                     vocab.append(han)
  178.         return vocab
  179.  
  180.     def ctc_len(self, label):
  181.         add_len = 0
  182.         label_len = len(label)
  183.         for i in range(label_len - 1):
  184.             if label[i] == label[i + 1]:
  185.                 add_len += 1
  186.         return label_len + add_len
  187.  
  188. def compute_mfcc(file):
  189.     fs, audio = wav.read(file)
  190.     mfcc_feat = mfcc(audio, samplerate=fs, numcep=26)
  191.     mfcc_feat = mfcc_feat[::3]
  192.     mfcc_feat = np.transpose(mfcc_feat)
  193.     return mfcc_feat
  194.  
  195. def compute_fbank(file):
  196.     x = np.linspace(0, 400 - 1, 400, dtype=np.int64)
  197.     w = 0.54 - 0.46 * np.cos(2 * np.pi * (x) / (400 - 1))  
  198.     fs, wavsignal = wav.read(file)
  199.     time_window = 25  
  200.     wav_arr = np.array(wavsignal)
  201.     range0_end = int(len(wavsignal) / fs * 1000 - time_window) // 10 + 1
  202.     data_input = np.zeros((range0_end, 200), dtype=np.float) 
  203.     data_line = np.zeros((1, 400), dtype=np.float)
  204.     for i in range(0, range0_end):
  205.         p_start = i * 160
  206.         p_end = p_start + 400
  207.         data_line = wav_arr[p_start:p_end]
  208.         data_line = data_line * w 
  209.         data_line = np.abs(fft(data_line))
  210.         data_input[i] = data_line[0:200]  
  211.     data_input = np.log(data_input + 1)
  212.     return data_input
  213.  
  214.  
  215. # word error rate------------------------------------
  216. def GetEditDistance(str1, str2):
  217. 	leven_cost = 0
  218. 	s = difflib.SequenceMatcher(None, str1, str2)
  219. 	for tag, i1, i2, j1, j2 in s.get_opcodes():
  220. 		if tag == 'replace':
  221. 			leven_cost += max(i2-i1, j2-j1)
  222. 		elif tag == 'insert':
  223. 			leven_cost += (j2-j1)
  224. 		elif tag == 'delete':
  225. 			leven_cost += (i2-i1)
  226. 	return leven_cost
  227.  
  228. # 解码器------------------------------------
  229. def decode_ctc(num_result, num2word):
  230. 	result = num_result[:, :, :]
  231. 	in_len = np.zeros((1), dtype = np.int32)
  232. 	in_len[0] = result.shape[1]
  233. 	r = K.ctc_decode(result, in_len, greedy = True, beam_width=10, top_paths=1)
  234. 	r1 = K.get_value(r[0][0])
  235. 	r1 = r1[0]
  236. 	text = []
  237. 	for i in r1:
  238. 		text.append(num2word[i])
  239. 	return r1, text

声学模型

  1. from keras.layers import Input, Conv2D, BatchNormalization, MaxPooling2D
  2. from keras.layers import Reshape, Dense, Dropout, Lambda
  3. from keras.optimizers import Adam
  4. from keras import backend as K
  5. from keras.models import Model
  6. from keras.utils import multi_gpu_model
  7. import tensorflow as tf
  8.  
  9. def am_hparams():
  10.     params = tf.contrib.training.HParams(
  11.         
  12.         vocab_size=50,
  13.         lr=0.0008,
  14.         gpu_nums=1,
  15.         is_training=True)
  16.     return params
  17.  
  18.  
  19. # =============================搭建模型====================================
  20. class Am():
  21.     """docstring for Amodel."""
  22.     def __init__(self, args):
  23.         self.vocab_size = args.vocab_size
  24.         self.gpu_nums = args.gpu_nums
  25.         self.lr = args.lr
  26.         self.is_training = args.is_training
  27.         self._model_init()
  28.         if self.is_training:
  29.             self._ctc_init()
  30.             self.opt_init()
  31.  
  32.     def _model_init(self):
  33.         self.inputs = Input(name='the_inputs', shape=(None, 200, 1))
  34.         self.h1 = cnn_cell(32, self.inputs)
  35.         self.h2 = cnn_cell(64, self.h1)
  36.         self.h3 = cnn_cell(128, self.h2)
  37.         self.h4 = cnn_cell(128, self.h3, pool=False)
  38.         self.h5 = cnn_cell(128, self.h4, pool=False)
  39.         # 200 / 8 * 128 = 3200
  40.         self.h6 = Reshape((-1, 3200))(self.h5)
  41.         self.h6 = Dropout(0.2)(self.h6)
  42.         self.h7 = dense(256)(self.h6)
  43.         self.h7 = Dropout(0.2)(self.h7)
  44.         self.outputs = dense(self.vocab_size, activation='softmax')(self.h7)
  45.         self.model = Model(inputs=self.inputs, outputs=self.outputs)
  46.         self.model.summary()
  47.  
  48.     def _ctc_init(self):
  49.         self.labels = Input(name='the_labels', shape=[None], dtype='float32')
  50.         self.input_length = Input(name='input_length', shape=[1], dtype='int64')
  51.         self.label_length = Input(name='label_length', shape=[1], dtype='int64')
  52.         self.loss_out = Lambda(ctc_lambda, output_shape=(1,), name='ctc')\
  53.             ([self.labels, self.outputs, self.input_length, self.label_length])
  54.         self.ctc_model = Model(inputs=[self.labels, self.inputs,
  55.             self.input_length, self.label_length], outputs=self.loss_out)
  56.  
  57.     def opt_init(self):
  58.         opt = Adam(lr = self.lr, beta_1 = 0.9, beta_2 = 0.999, decay = 0.01, epsilon = 10e-8)
  59.         if self.gpu_nums > 1:
  60.             self.ctc_model=multi_gpu_model(self.ctc_model,gpus=self.gpu_nums)
  61.         self.ctc_model.compile(loss={'ctc': lambda y_true, output: output}, optimizer=opt)
  62.  
  63. # ============================模型组件=================================
  64. def conv2d(size):
  65.     return Conv2D(size, (3,3), use_bias=True, activation='relu',
  66.         padding='same', kernel_initializer='he_normal')
  67.  
  68.  
  69. def norm(x):
  70.     return BatchNormalization(axis=-1)(x)
  71.  
  72.  
  73. def maxpool(x):
  74.     return MaxPooling2D(pool_size=(2,2), strides=None, padding="valid")(x)
  75.  
  76. def dense(units, activation="relu"):
  77.     return Dense(units, activation=activation, use_bias=True,
  78.         kernel_initializer='he_normal')
  79.  
  80.  
  81. # x.shape=(none, none, none)
  82. # output.shape = (1/2, 1/2, 1/2)
  83. def cnn_cell(size, x, pool=True):
  84.     x = norm(conv2d(size)(x))
  85.     x = norm(conv2d(size)(x))
  86.     if pool:
  87.         x = maxpool(x)
  88.     return x
  89.  
  90. def ctc_lambda(args):
  91.     labels, y_pred, input_length, label_length = args
  92.     y_pred = y_pred[:, :, :]
  93.     return K.ctc_batch_cost(labels, y_pred, input_length, label_length)

语言模型

  1. import tensorflow as tf
  2. import numpy as np
  3. def normalize(inputs, 
  4.               epsilon = 1e-8,
  5.               scope="ln",
  6.               reuse=None):
  7.     
  8.     with tf.variable_scope(scope, reuse=reuse):
  9.         inputs_shape = inputs.get_shape()
  10.         params_shape = inputs_shape[-1:]
  11.  
  12.         mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)
  13.         beta= tf.Variable(tf.zeros(params_shape))
  14.         gamma = tf.Variable(tf.ones(params_shape))
  15.         normalized = (inputs - mean) / ( (variance + epsilon) ** (.5) )
  16.         outputs = gamma * normalized + beta
  17.     return outputs
  18.  
  19. def embedding(inputs, 
  20.               vocab_size, 
  21.               num_units, 
  22.               zero_pad=True, 
  23.               scale=True,
  24.               scope="embedding", 
  25.               reuse=None):
  26.     
  27.     with tf.variable_scope(scope, reuse=reuse):
  28.         lookup_table = tf.get_variable('lookup_table',
  29.                                        dtype=tf.float32,
  30.                                        shape=[vocab_size, num_units],
  31.                                        initializer=tf.contrib.layers.xavier_initializer())  
  32.         if zero_pad:
  33.             lookup_table = tf.concat((tf.zeros(shape=[1, num_units]), lookup_table[1:, :]), 0)   
  34.         outputs = tf.nn.embedding_lookup(lookup_table, inputs)  
  35.  
  36.         if scale:
  37.             outputs = outputs * (num_units ** 0.5) 
  38.  
  39.     return outputs
  40.  
  41.  
  42. def multihead_attention(emb,
  43.                         queries, 
  44.                         keys, 
  45.                         num_units=None, 
  46.                         num_heads=8, 
  47.                         dropout_rate=0,
  48.                         is_training=True,
  49.                         causality=False,
  50.                         scope="multihead_attention", 
  51.                         reuse=None):
  52.     
  53.     with tf.variable_scope(scope, reuse=reuse):
  54.         # Set the fall back option for num_units
  55.         if num_units is None:
  56.             num_units = queries.get_shape().as_list[-1]
  57.         
  58.         # Linear projections
  59.         Q = tf.layers.dense(queries, num_units, activation=tf.nn.relu) # (N, T_q, C)
  60.         K = tf.layers.dense(keys, num_units, activation=tf.nn.relu) # (N, T_k, C)
  61.         V = tf.layers.dense(keys, num_units, activation=tf.nn.relu) # (N, T_k, C)
  62.         
  63.         # Split and concat
  64.         Q_ = tf.concat(tf.split(Q, num_heads, axis=2), axis=0) # (h*N, T_q, C/h) 
  65.         K_ = tf.concat(tf.split(K, num_heads, axis=2), axis=0) # (h*N, T_k, C/h) 
  66.         V_ = tf.concat(tf.split(V, num_heads, axis=2), axis=0) # (h*N, T_k, C/h) 
  67.  
  68.         # Multiplication
  69.         outputs = tf.matmul(Q_, tf.transpose(K_, [0, 2, 1])) # (h*N, T_q, T_k)
  70.         
  71.         # Scale
  72.         outputs = outputs / (K_.get_shape().as_list()[-1] ** 0.5)
  73.         
  74.         # Key Masking
  75.         key_masks = tf.sign(tf.abs(tf.reduce_sum(emb, axis=-1))) # (N, T_k)
  76.         key_masks = tf.tile(key_masks, [num_heads, 1]) # (h*N, T_k)
  77.         key_masks = tf.tile(tf.expand_dims(key_masks, 1), [1, tf.shape(queries)[1], 1]) # (h*N, T_q, T_k)
  78.         
  79.         paddings = tf.ones_like(outputs)*(-2**32+1)
  80.         outputs = tf.where(tf.equal(key_masks, 0), paddings, outputs) # (h*N, T_q, T_k)
  81.   
  82.         # Causality = Future blinding
  83.         if causality:
  84.             diag_vals = tf.ones_like(outputs[0, :, :]) # (T_q, T_k)
  85.             tril = tf.contrib.linalg.LinearOperatorTriL(diag_vals).to_dense() # (T_q, T_k)
  86.             masks = tf.tile(tf.expand_dims(tril, 0), [tf.shape(outputs)[0], 1, 1]) # (h*N, T_q, T_k)
  87.    
  88.             paddings = tf.ones_like(masks)*(-2**32+1)
  89.             outputs = tf.where(tf.equal(masks, 0), paddings, outputs) # (h*N, T_q, T_k)
  90.   
  91.         # Activation
  92.         outputs = tf.nn.softmax(outputs) # (h*N, T_q, T_k)
  93.          
  94.         # Query Masking
  95.         query_masks = tf.sign(tf.abs(tf.reduce_sum(emb, axis=-1))) # (N, T_q)
  96.         query_masks = tf.tile(query_masks, [num_heads, 1]) # (h*N, T_q)
  97.         query_masks = tf.tile(tf.expand_dims(query_masks, -1), [1, 1, tf.shape(keys)[1]]) # (h*N, T_q, T_k)
  98.         outputs *= query_masks # broadcasting. (N, T_q, C)
  99.           
  100.         # Dropouts
  101.         outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=tf.convert_to_tensor(is_training))
  102.                
  103.         # Weighted sum
  104.         outputs = tf.matmul(outputs, V_) # ( h*N, T_q, C/h)
  105.         
  106.         # Restore shape
  107.         outputs = tf.concat(tf.split(outputs, num_heads, axis=0), axis=2 ) # (N, T_q, C)
  108.               
  109.         # Residual connection
  110.         outputs += queries
  111.               
  112.         # Normalize
  113.         outputs = normalize(outputs) # (N, T_q, C)
  114.  
  115.     return outputs
  116.     
  117.  
  118. def feedforward(inputs, 
  119.                 num_units=[2048, 512],
  120.                 scope="multihead_attention", 
  121.                 reuse=None):
  122.     
  123.     with tf.variable_scope(scope, reuse=reuse):
  124.         # Inner layer
  125.         params = {"inputs": inputs, "filters": num_units[0], "kernel_size": 1,
  126.                   "activation": tf.nn.relu, "use_bias": True}
  127.         outputs = tf.layers.conv1d(**params)
  128.         
  129.         # Readout layer
  130.         params = {"inputs": outputs, "filters": num_units[1], "kernel_size": 1,
  131.                   "activation": None, "use_bias": True}
  132.         outputs = tf.layers.conv1d(**params)
  133.         
  134.         # Residual connection
  135.         outputs += inputs
  136.         
  137.         # Normalize
  138.         outputs = normalize(outputs)
  139.     
  140.     return outputs
  141.  
  142.  
  143. def label_smoothing(inputs, epsilon=0.1):
  144.     
  145.     K = inputs.get_shape().as_list()[-1] # number of channels
  146.     return ((1-epsilon) * inputs) + (epsilon / K)
  147.  
  148.  
  149. class Lm():
  150.     '''
  151.     #   语言模型
  152.     '''
  153.     def __init__(self, arg):
  154.         self.graph = tf.Graph()
  155.         with self.graph.as_default():
  156.             self.is_training = arg.is_training
  157.             self.hidden_units = arg.hidden_units
  158.             self.input_vocab_size = arg.input_vocab_size
  159.             self.label_vocab_size = arg.label_vocab_size
  160.             self.num_heads = arg.num_heads
  161.             self.num_blocks = arg.num_blocks
  162.             self.max_length = arg.max_length
  163.             self.lr = arg.lr
  164.             self.dropout_rate = arg.dropout_rate
  165.                 
  166.             # input
  167.             self.x = tf.placeholder(tf.int32, shape=(None, None))
  168.             self.y = tf.placeholder(tf.int32, shape=(None, None))
  169.             # embedding
  170.             self.emb = embedding(self.x, vocab_size=self.input_vocab_size, num_units=self.hidden_units, scale=True, scope="enc_embed")
  171.             # 编码模块
  172.             self.enc = self.emb + embedding(tf.tile(tf.expand_dims(tf.range(tf.shape(self.x)[1]), 0), [tf.shape(self.x)[0], 1]), vocab_size=self.max_length,num_units=self.hidden_units, zero_pad=False, scale=False,scope="enc_pe")
  173.             ## Dropout
  174.             self.enc = tf.layers.dropout(self.enc, 
  175.                                         rate=self.dropout_rate, 
  176.                                         training=tf.convert_to_tensor(self.is_training))
  177.                         
  178.             ## Blocks
  179.             for i in range(self.num_blocks):
  180.                 with tf.variable_scope("num_blocks_{}".format(i)):
  181.                     ### Multihead Attention
  182.                     self.enc = multihead_attention(emb = self.emb,
  183.                                                     queries=self.enc, 
  184.                                                     keys=self.enc, 
  185.                                                     num_units=self.hidden_units, 
  186.                                                     num_heads=self.num_heads, 
  187.                                                     dropout_rate=self.dropout_rate,
  188.                                                     is_training=self.is_training,
  189.                                                     causality=False)
  190.                                 
  191.             ### Feed Forward
  192.             self.outputs = feedforward(self.enc, num_units=[4*self.hidden_units, self.hidden_units])
  193.                                     
  194.             # Final linear projection
  195.             self.logits = tf.layers.dense(self.outputs, self.label_vocab_size)
  196.             self.preds = tf.to_int32(tf.argmax(self.logits, axis=-1))
  197.             self.istarget = tf.to_float(tf.not_equal(self.y, 0))
  198.             self.acc = tf.reduce_sum(tf.to_float(tf.equal(self.preds, self.y))*self.istarget)/ (tf.reduce_sum(self.istarget))
  199.             tf.summary.scalar('acc', self.acc)
  200.                         
  201.             if self.is_training:  
  202.                 # Loss
  203.                 self.y_smoothed = label_smoothing(tf.one_hot(self.y, depth=self.label_vocab_size))
  204.                 self.loss = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.y_smoothed)
  205.                 self.mean_loss = tf.reduce_sum(self.loss*self.istarget) / (tf.reduce_sum(self.istarget))
  206.                 
  207.                 # Training Scheme
  208.                 self.global_step = tf.Variable(0, name='global_step', trainable=False)
  209.                 self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.9, beta2=0.98, epsilon=1e-8)
  210.                 self.train_op = self.optimizer.minimize(self.mean_loss, global_step=self.global_step)
  211.                         
  212.                 # Summary 
  213.                 tf.summary.scalar('mean_loss', self.mean_loss)
  214.                 self.merged = tf.summary.merge_all()
  215.  
  216.  
  217. def lm_hparams():
  218.     params = tf.contrib.training.HParams(
  219.         num_heads = 8,
  220.         num_blocks = 6,
  221.         # vocab
  222.         input_vocab_size = 50,
  223.         label_vocab_size = 50,
  224.         # embedding size
  225.         max_length = 100,
  226.         hidden_units = 512,
  227.         dropout_rate = 0.2,
  228.         lr = 0.0003,
  229.         is_training = True)
  230.     return params

模型测试代码

  1. #coding=utf-8
  2. import tensorflow as tf
  3. import numpy as np
  4. from utils import decode_ctc, GetEditDistance
  5.  
  6.  
  7. # 0.准备解码所需字典，参数需和训练一致，也可以将字典保存到本地，直接进行读取
  8. from utils import get_data, data_hparams
  9. data_args = data_hparams()
  10. train_data = get_data(data_args)
  11.  
  12.  
  13. # 1.声学模型-----------------------------------
  14. from model_speech.cnn_ctc import Am, am_hparams
  15.  
  16. am_args = am_hparams()
  17. am_args.vocab_size = len(train_data.am_vocab)
  18. am = Am(am_args)
  19. print('loading acoustic model...')
  20. am.ctc_model.load_weights('logs_am/model.h5')
  21.  
  22. # 2.语言模型-------------------------------------------
  23. from model_language.transformer import Lm, lm_hparams
  24.  
  25. lm_args = lm_hparams()
  26. lm_args.input_vocab_size = len(train_data.pny_vocab)
  27. lm_args.label_vocab_size = len(train_data.han_vocab)
  28. lm_args.dropout_rate = 0.
  29. print('loading language model...')
  30. lm = Lm(lm_args)
  31. sess = tf.Session(graph=lm.graph)
  32. with lm.graph.as_default():
  33.     saver =tf.train.Saver()
  34. with sess.as_default():
  35.     latest = tf.train.latest_checkpoint('logs_lm')
  36.     saver.restore(sess, latest)
  37.  
  38. # 3. 准备测试所需数据， 不必和训练数据一致，通过设置data_args.data_type测试，
  39. #    此处应设为'test'，我用了'train'因为演示模型较小，如果使用'test'看不出效果，
  40. #    且会出现未出现的词。
  41. data_args.data_type = 'train'
  42. data_args.shuffle = False
  43. data_args.batch_size = 1
  44. test_data = get_data(data_args)
  45.  
  46. # 4. 进行测试-------------------------------------------
  47. am_batch = test_data.get_am_batch()
  48. word_num = 0
  49. word_error_num = 0
  50. for i in range(8):
  51.     print('\n the ', i, 'th example.')
  52.     
  53.     inputs, _ = next(am_batch)
  54.     x = inputs['the_inputs']
  55.     y = test_data.pny_lst[i]
  56.     result = am.model.predict(x, steps=1)
  57.     
  58.     _, text = decode_ctc(result, train_data.am_vocab)
  59.     text = ' '.join(text)
  60.     print('文本结果：', text)
  61.     print('原文结果：', ' '.join(y))
  62.     with sess.as_default():
  63.         text = text.strip('\n').split(' ')
  64.         x = np.array([train_data.pny_vocab.index(pny) for pny in text])
  65.         x = x.reshape(1, -1)
  66.         preds = sess.run(lm.preds, {lm.x: x})
  67.         label = test_data.han_lst[i]
  68.         got = ''.join(train_data.han_vocab[idx] for idx in preds[0])
  69.         print('原文汉字：', label)
  70.         print('识别结果：', got)
  71.         word_error_num += min(len(label), GetEditDistance(label, got))
  72.         word_num += len(label)
  73. print('词错误率：', word_error_num / word_num)
  74. sess.close()

模型测试结果

  1.  the  0 th example.
  2. 文本结果： lv4 shi4 yang2 chun1 yan1 jing3 da4 kuai4 wen2 zhang1 de di3 se4 si4 yue4 de lin2 luan2 geng4 shi4 lv4 de2 xian1 huo2 xiu4 mei4 shi1 yi4 ang4 ran2
  3. 原文结果： lv4 shi4 yang2 chun1 yan1 jing3 da4 kuai4 wen2 zhang1 de di3 se4 si4 yue4 de lin2 luan2 geng4 shi4 lv4 de2 xian1 huo2 xiu4 mei4 shi1 yi4 ang4 ran2
  4. 原文汉字： 绿是阳春烟景大块文章的底色四月的林峦更是绿得鲜活秀媚诗意盎然
  5. 识别结果： 绿是阳春烟景大块文章的底色四月的林峦更是绿得鲜活秀媚诗意盎然
  6.  
  7.  the  1 th example.
  8. 文本结果： ta1 jin3 ping2 yao1 bu4 de li4 liang4 zai4 yong3 dao4 shang4 xia4 fan1 teng2 yong3 dong4 she2 xing2 zhuang4 ru2 hai3 tun2 yi4 zhi2 yi3 yi1 tou2 de you1 shi4 ling3 xian1
  9. 原文结果： ta1 jin3 ping2 yao1 bu4 de li4 liang4 zai4 yong3 dao4 shang4 xia4 fan1 teng2 yong3 dong4 she2 xing2 zhuang4 ru2 hai3 tun2 yi4 zhi2 yi3 yi1 tou2 de you1 shi4 ling3 xian1
  10. 原文汉字： 他仅凭腰部的力量在泳道上下翻腾蛹动蛇行状如海豚一直以一头的优势领先
  11. 识别结果： 他仅凭腰部的力量在蛹道上下翻腾蛹动蛇行状如海豚一直以一头的优势领先
  12.  
  13.  the  2 th example.
  14. 文本结果： qi3 ye4 yi1 kao4 ji4 shu4 wa1 qian2 zeng1 xiao4 ta1 fu4 ze2 quan2 chang3 chan3 pin3 zhi4 liang4 yu3 ji4 shu4 pei2 xun4 cheng2 le chang3 li3 de da4 mang2 ren2
  15. 原文结果： qi3 ye4 yi1 kao4 ji4 shu4 wa1 qian2 zeng1 xiao4 ta1 fu4 ze2 quan2 chang3 chan3 pin3 zhi4 liang4 yu3 ji4 shu4 pei2 xun4 cheng2 le chang3 li3 de da4 mang2 ren2
  16. 原文汉字： 企业依靠技术挖潜增效他负责全厂产品质量与技术培训成了厂里的大忙人
  17. 识别结果： 企业依靠技术挖潜增效他负责全厂产品质量与技术培训成了厂里的大忙人
  18.  
  19.  the  3 th example.
  20. 文本结果： cai4 zuo4 hao3 le yi1 wan3 qing1 zheng1 wu3 chang1 yu2 yi1 wan3 fan1 jia1 chao3 ji1 dan4 yi1 wan3 zha4 cai4 gan4 zi chao3 rou4 si1
  21. 原文结果： cai4 zuo4 hao3 le yi1 wan3 qing1 zheng1 wu3 chang1 yu2 yi1 wan3 fan1 jia1 chao3 ji1 dan4 yi1 wan3 zha4 cai4 gan4 zi chao3 rou4 si1
  22. 原文汉字： 菜做好了一碗清蒸武昌鱼一碗蕃茄炒鸡蛋一碗榨菜干子炒肉丝
  23. 识别结果： 菜做好了一碗清蒸武昌鱼一碗蕃茄炒鸡蛋一碗榨菜干子炒肉丝
  24.  
  25.  the  4 th example.
  26. 文本结果： ta1 kan4 kan4 ye4 ji3 hen3 shen1 bai2 tian1 de yan2 re4 yi3 gei3 ye4 liang2 chui1 san4 fen1 fu4 da4 jia1 ge4 zi4 an1 xi1 ming2 tian1 ji4 xu4 wan2 le4
  27. 原文结果： ta1 kan4 kan4 ye4 ji3 hen3 shen1 bai2 tian1 de yan2 re4 yi3 gei3 ye4 liang2 chui1 san4 fen1 fu4 da4 jia1 ge4 zi4 an1 xi1 ming2 tian1 ji4 xu4 wan2 le4
  28. 原文汉字： 她看看夜己很深白天的炎热已给夜凉吹散吩咐大家各自安息明天继续玩乐
  29. 识别结果： 她看看夜己很深白天的炎热已给夜凉吹散吩咐大家各自安息明天继续玩乐

项目文件的下载地址为：DeepSpeechRecognition.rar-深度学习文档类资源-CSDN下载

数据为 data文件夹，将data文件夹解压后放置在项目文件的根目录下即可以运行项目程序

数据的下载地址为：深度学习语音识别数据集data.rar-深度学习文档类资源-CSDN下载

声明：本文内容由网友自发贡献，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：【wpsshop博客】
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号