使用Tensorflow识别语音关键词

(py3-env) lili@lili-Precision-7720:~/data/speech_dataset$ ls
_background_noise_  down     go       marvin  README.md                     stop              validation_list.txt
backward            eight    happy    nine    right                         testing_list.txt  visual
bed                 five     house    no      seven                         three             wow
bird                follow   learn    off     sheila                        tree              yes
cat                 forward  left     on      six                           two               zero
dog                 four     LICENSE  one     speech_commands_v0.02.tar.gz  up

(py3-env) lili@lili-Precision-7720:~/data/speech_dataset$ ls one/|head
00176480_nohash_0.wav
004ae714_nohash_0.wav
00f0204f_nohash_0.wav
0132a06d_nohash_0.wav
0132a06d_nohash_1.wav
0132a06d_nohash_2.wav
0132a06d_nohash_3.wav
0132a06d_nohash_4.wav
0135f3f2_nohash_0.wav
0135f3f2_nohash_1.wav

(py3-env) lili@lili-Precision-7720:~/data/speech_dataset$ ls _background_noise_/
doing_the_dishes.wav  exercise_bike.wav  README.md        white_noise.wav
dude_miaowing.wav     pink_noise.wav     running_tap.wav

  desired_samples = int(sample_rate * clip_duration_ms / 1000)
  window_size_samples = int(sample_rate * window_size_ms / 1000)
  window_stride_samples = int(sample_rate * window_stride_ms / 1000)
  length_minus_window = (desired_samples - window_size_samples)
  if length_minus_window < 0:
	  spectrogram_length = 0
  else:
	  spectrogram_length = 1 + int(length_minus_window / window_stride_samples)
  if preprocess == 'average':
	  fft_bin_count = 1 + (_next_power_of_two(window_size_samples) / 2)
	  average_window_width = int(math.floor(fft_bin_count / feature_bin_count))
	  fingerprint_width = int(math.ceil(fft_bin_count / average_window_width))
  elif preprocess == 'mfcc':
	  average_window_width = -1
	  fingerprint_width = feature_bin_count
  else:
	  raise ValueError('Unknown preprocess mode "%s" (should be "mfcc" or'
		  ' "average")' % (preprocess))
  fingerprint_size = fingerprint_width * spectrogram_length

  def __init__(self, data_url, data_dir, silence_percentage, unknown_percentage,
		  wanted_words, validation_percentage, testing_percentage,
		  model_settings, summaries_dir):
	  self.data_dir = data_dir
	  self.maybe_download_and_extract_dataset(data_url, data_dir)
	  self.prepare_data_index(silence_percentage, unknown_percentage,
	  wanted_words, validation_percentage,
	  testing_percentage)
	  self.prepare_background_data()
	  self.prepare_processing_graph(model_settings, summaries_dir)

{'label': 'right', 'file': '/home/lili/data/speech_dataset/right/c22d3f18_nohash_2.wav'}

<class 'list'>: ['_silence_', '_unknown_', 'yes', 'no', 'up', 'down', 'left', 'right', 'on', 'off', 'stop', 'go']

{'marvin': 1, 'tree': 1, 'learn': 1, 'dog': 1, 'sheila': 1, 'bird': 1, 'right': 7, 'off': 9, 'backward': 1, 'six': 1, 'two': 1, 'no': 3, 'yes': 2, 'one': 1, 'follow': 1, 'up': 4, 'three': 1, 'forward': 1, 'happy': 1, 'nine': 1, 'bed': 1, 'zero': 1, 'house': 1, 'visual': 1, 'five': 1, 'seven': 1, 'cat': 1, 'left': 6, 'stop': 10, 'go': 11, 'four': 1, 'on': 8, 'wow': 1, 'down': 5, 'eight': 1}

  hash_name_hashed = hashlib.sha1(compat.as_bytes(hash_name)).hexdigest()
  percentage_hash = ((int(hash_name_hashed, 16) %
	  (MAX_NUM_WAVS_PER_CLASS + 1)) *
	  (100.0 / MAX_NUM_WAVS_PER_CLASS))
  if percentage_hash < validation_percentage:
	  result = 'validation'
  elif percentage_hash < (testing_percentage + validation_percentage):
	  result = 'testing'
  else:
	  result = 'training'
  return result

    # 我们需要一个文件来表示silence，因为之后会乘以0，所以它的内容不重要。我们这里选择训练数据的第一个文件。
    silence_wav_path = self.data_index['training'][0]['file']
    for set_index in ['validation', 'testing', 'training']:
	    set_size = len(self.data_index[set_index])
	    silence_size = int(math.ceil(set_size * silence_percentage / 100))
	    for _ in range(silence_size):
		    self.data_index[set_index].append({
			    'label': SILENCE_LABEL,
			    'file': silence_wav_path
		    })
	    # 随机寻找相应数量的unknown词。
	    random.shuffle(unknown_index[set_index])
	    unknown_size = int(math.ceil(set_size * unknown_percentage / 100))
	    self.data_index[set_index].extend(unknown_index[set_index][:unknown_size])

    with tf.Session(graph=tf.Graph()) as sess:
	    wav_filename_placeholder = tf.placeholder(tf.string, [])
	    wav_loader = io_ops.read_file(wav_filename_placeholder)
	    wav_decoder = contrib_audio.decode_wav(wav_loader, desired_channels=1)
	    search_path = os.path.join(self.data_dir, BACKGROUND_NOISE_DIR_NAME,
		    '*.wav')
	    for wav_path in gfile.Glob(search_path):
		    wav_data = sess.run(
			    wav_decoder,
			    feed_dict={wav_filename_placeholder: wav_path}).audio.flatten()
		    self.background_data.append(wav_data)

    with tf.get_default_graph().name_scope('data'):
	    desired_samples = model_settings['desired_samples']
	    self.wav_filename_placeholder_ = tf.placeholder(
		    tf.string, [], name='wav_filename')
	    wav_loader = io_ops.read_file(self.wav_filename_placeholder_)
	    wav_decoder = contrib_audio.decode_wav(
		    wav_loader, desired_channels=1, desired_samples=desired_samples)
	    # Allow the audio sample's volume to be adjusted.
	    self.foreground_volume_placeholder_ = tf.placeholder(
		    tf.float32, [], name='foreground_volume')
	    scaled_foreground = tf.multiply(wav_decoder.audio,
		    self.foreground_volume_placeholder_)
	    # Shift the sample's start position, and pad any gaps with zeros.
	    self.time_shift_padding_placeholder_ = tf.placeholder(
		    tf.int32, [2, 2], name='time_shift_padding')
	    self.time_shift_offset_placeholder_ = tf.placeholder(
		    tf.int32, [2], name='time_shift_offset')
	    padded_foreground = tf.pad(
		    scaled_foreground,
		    self.time_shift_padding_placeholder_,
		    mode='CONSTANT')
	    sliced_foreground = tf.slice(padded_foreground,
		    self.time_shift_offset_placeholder_,
		    [desired_samples, -1])
	    # Mix in background noise.
	    self.background_data_placeholder_ = tf.placeholder(
		    tf.float32, [desired_samples, 1], name='background_data')
	    self.background_volume_placeholder_ = tf.placeholder(
		    tf.float32, [], name='background_volume')
	    background_mul = tf.multiply(self.background_data_placeholder_,
		    self.background_volume_placeholder_)
	    background_add = tf.add(background_mul, sliced_foreground)
	    background_clamp = tf.clip_by_value(background_add, -1.0, 1.0)
	    # Run the spectrogram and MFCC ops to get a 2D 'fingerprint' of the audio.
	    spectrogram = contrib_audio.audio_spectrogram(
		    background_clamp,
		    window_size=model_settings['window_size_samples'],
		    stride=model_settings['window_stride_samples'],
		    magnitude_squared=True)
	    tf.summary.image(
		    'spectrogram', tf.expand_dims(spectrogram, -1), max_outputs=1)

	    if model_settings['preprocess'] == 'average':
		    self.output_ = tf.nn.pool(
		    tf.expand_dims(spectrogram, -1),
		    window_shape=[1, model_settings['average_window_width']],
		    strides=[1, model_settings['average_window_width']],
			    pooling_type='AVG',
			    padding='SAME')
		    tf.summary.image('shrunk_spectrogram', self.output_, max_outputs=1)
	    elif model_settings['preprocess'] == 'mfcc':
		    self.output_ = contrib_audio.mfcc(
			    spectrogram,
			    wav_decoder.sample_rate,
			    dct_coefficient_count=model_settings['fingerprint_width'])
		    tf.summary.image(
			    'mfcc', tf.expand_dims(self.output_, -1), max_outputs=1)
	    else:
		    raise ValueError('Unknown preprocess mode "%s" (should be "mfcc" or'
			    ' "average")' % (model_settings['preprocess']))
	    
	    # Merge all the summaries and write them out to /tmp/retrain_logs (by
	    # default)
	    self.merged_summaries_ = tf.summary.merge_all(scope='data')
	    self.summary_writer_ = tf.summary.FileWriter(summaries_dir + '/data',
	    tf.get_default_graph())

  input_placeholder = tf.placeholder(
	  tf.float32, [None, fingerprint_size], name='fingerprint_input')
	  
  fingerprint_input = input_placeholder
  logits, dropout_prob = models.create_model(
	  fingerprint_input,
	  model_settings,
	  FLAGS.model_architecture,
	  is_training=True)

def create_conv_model(fingerprint_input, model_settings, is_training):
	"""构建一个标准的卷积网络
	
	这个网络结果和下面论文'Convolutional Neural Networks for Small-footprint Keyword Spotting'
        的'cnn-trad-fpool3'类似，论文的url是：
	
	http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf
	
	这是graph的结构：
	
	(fingerprint_input)
		v
		[Conv2D]<-(weights)
		v
		[BiasAdd]<-(bias)
		v
		[Relu]
		v
		[MaxPool]
		v
		[Conv2D]<-(weights)
		v
		[BiasAdd]<-(bias)
		v
		[Relu]
		v
		[MaxPool]
		v
		[MatMul]<-(weights)
		v
		[BiasAdd]<-(bias)
		v
	
	这个模型的效果很不错，但是它有大量的参数(占内存)和需要大量的计算(费CPU)。
        如果需要在资源不足的环境里，建议使用'low_latency_conv'
	
	如果is_training是True，那么会返回一个dropout的PlaceHolder
        ，调用者在session.run的时候需要feed需要dropout的比例。
	
	
	参数:
		fingerprint_input: 输入Tensor，(batch, fingerprint_size) 
		model_settings: 模型的设置
		is_training: 是否训练阶段
		
	返回值:
		输出分类的logits，如果是训练，还会返回一个dropout的place holder
	"""
	if is_training:
		dropout_prob = tf.placeholder(tf.float32, name='dropout_prob')
	input_frequency_size = model_settings['fingerprint_width']
	input_time_size = model_settings['spectrogram_length']
	fingerprint_4d = tf.reshape(fingerprint_input,
		[-1, input_time_size, input_frequency_size, 1])
	first_filter_width = 8
	first_filter_height = 20
	first_filter_count = 64
	first_weights = tf.get_variable(
		name='first_weights',
		initializer=tf.truncated_normal_initializer(stddev=0.01),
		shape=[first_filter_height, first_filter_width, 1, first_filter_count])
	first_bias = tf.get_variable(
		name='first_bias',
		initializer=tf.zeros_initializer,
		shape=[first_filter_count])
	first_conv = tf.nn.conv2d(fingerprint_4d, first_weights, [1, 1, 1, 1],
		'SAME') + first_bias
	first_relu = tf.nn.relu(first_conv)
	if is_training:
		first_dropout = tf.nn.dropout(first_relu, dropout_prob)
	else:
		first_dropout = first_relu
	max_pool = tf.nn.max_pool(first_dropout, [1, 2, 2, 1], [1, 2, 2, 1], 'SAME')
	
	
	second_filter_width = 4
	second_filter_height = 10
	second_filter_count = 64
	second_weights = tf.get_variable(
	name='second_weights',
	initializer=tf.truncated_normal_initializer(stddev=0.01),
		shape=[
		second_filter_height, second_filter_width, first_filter_count,
		second_filter_count
		])
	second_bias = tf.get_variable(
		name='second_bias',
		initializer=tf.zeros_initializer,
		shape=[second_filter_count])
	second_conv = tf.nn.conv2d(max_pool, second_weights, [1, 1, 1, 1],
		'SAME') + second_bias
	second_relu = tf.nn.relu(second_conv)
	if is_training:
		second_dropout = tf.nn.dropout(second_relu, dropout_prob)
	else:
		second_dropout = second_relu
		
	second_conv_shape = second_dropout.get_shape()
	second_conv_output_width = second_conv_shape[2]
	second_conv_output_height = second_conv_shape[1]
	second_conv_element_count = int(
		second_conv_output_width * second_conv_output_height *
		second_filter_count)
	flattened_second_conv = tf.reshape(second_dropout,
		[-1, second_conv_element_count])
	label_count = model_settings['label_count']
	final_fc_weights = tf.get_variable(
		name='final_fc_weights',
		initializer=tf.truncated_normal_initializer(stddev=0.01),
		shape=[second_conv_element_count, label_count])
	final_fc_bias = tf.get_variable(
		name='final_fc_bias',
		initializer=tf.zeros_initializer,
		shape=[label_count])
	final_fc = tf.matmul(flattened_second_conv, final_fc_weights) + final_fc_bias
	if is_training:
		return final_fc, dropout_prob
	else:
		return final_fc

  for training_step in xrange(start_step, training_steps_max + 1):
	  # ... 更加step确定当前的learning rate ...
	  # 获取一个batch的数据
	  train_fingerprints, train_ground_truth = audio_processor.get_data(
		  FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency,
		  FLAGS.background_volume, time_shift_samples, 'training', sess)
	  # 运行一次训练
	  train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run(
		  [
			  merged_summaries,
			  evaluation_step,
			  cross_entropy_mean,
			  train_step,
			  increment_global_step,
		  ],
		  feed_dict={
			  fingerprint_input: train_fingerprints,
			  ground_truth_input: train_ground_truth,
			  learning_rate_input: learning_rate_value,
			  dropout_prob: 0.5
		  })

for i in xrange(offset, offset + sample_count):
    # 选择音频的下标，如果是确定的选取(比如非training阶段)
    if how_many == -1 or pick_deterministically:
	    sample_index = i
    else:
	    # 随机选择一个音频
	    sample_index = np.random.randint(len(candidates))
    sample = candidates[sample_index]
    
    # 如果参数time_shift > 0 则随机产生一个[-time_shift, time_shift]的平移量。
    if time_shift > 0:
	    time_shift_amount = np.random.randint(-time_shift, time_shift)
    else:
	    time_shift_amount = 0
	
		   
    if time_shift_amount > 0:
	    # 如果是右移，则在左边papdding 
	    time_shift_padding = [[time_shift_amount, 0], [0, 0]]
	    time_shift_offset = [0, 0]
    else:
	    # 如果是右移，则在右边padding，并且需要设置offset为平移量
	    time_shift_padding = [[0, -time_shift_amount], [0, 0]]
	    time_shift_offset = [-time_shift_amount, 0]
    
    # 运行session的feed dict
    input_dict = {
	    self.wav_filename_placeholder_: sample['file'],
	    self.time_shift_padding_placeholder_: time_shift_padding,
	    self.time_shift_offset_placeholder_: time_shift_offset,
    }
    # 如果需要加入背景噪音(训练阶段)或者是silence，那么需要加入噪音
    if use_background or sample['label'] == SILENCE_LABEL:
	    background_index = np.random.randint(len(self.background_data))
	    background_samples = self.background_data[background_index]
		
		# 从噪音文件里随机选择1s的数据。
	    background_offset = np.random.randint(
		    0, len(background_samples) - model_settings['desired_samples'])
		    background_clipped = background_samples[background_offset:(
		    background_offset + desired_samples)]
		    background_reshaped = background_clipped.reshape([desired_samples, 1])
		# 如果是silence，那么噪音的音量是随机的从(0, 1)中均匀选择
	    if sample['label'] == SILENCE_LABEL:
		    background_volume = np.random.uniform(0, 1)
		# 如果是加背景噪音，那么需要给background_frequency(0.8)比例的加噪音    
	    elif np.random.uniform(0, 1) < background_frequency:
		    #背景噪音的音量从(0, background_volume_range)中均匀产生。
		    background_volume = np.random.uniform(0, background_volume_range)
		# 1- background_frequency(0.2)比例的不加噪音    
	    else:
		    background_volume = 0
    else:
	    # 不用加噪音
	    background_reshaped = np.zeros([desired_samples, 1])
	    background_volume = 0
	
	# 设置feed dict的背景噪音和音量    
    input_dict[self.background_data_placeholder_] = background_reshaped
    input_dict[self.background_volume_placeholder_] = background_volume
    
    # 如果数据是silence，那么设置音量为0。
    if sample['label'] == SILENCE_LABEL:
	    input_dict[self.foreground_volume_placeholder_] = 0
    else:
	    input_dict[self.foreground_volume_placeholder_] = 1
	    
    # 运行graph来产生audio 
    summary, data_tensor = sess.run(
	    [self.merged_summaries_, self.output_], feed_dict=input_dict)
	    
    self.summary_writer_.add_summary(summary)
    data[i - offset, :] = data_tensor.flatten()
    label_index = self.word_to_index[sample['label']]
    labels[i - offset] = label_index
return data, labels \end{lstlisting}