devbox
天景科技苑
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

在finetune的时候优化bert中优化器AdamWeightDecayOptimizer

作者：天景科技苑 | 2024-08-08 04:16:18

赞

踩

adamweightdecayoptimizer

 主要是bert源码当中的adam是简化版本的，这里给出完整版的adamWeightDecay

  1. # bert源码中的AdamWeightDecayOptimizer
  2. class AdamWeightDecayOptimizer(tf.train.Optimizer):
  3.     """A basic Adam optimizer that includes "correct" L2 weight decay."""
  4.  
  5.     def __init__(self,
  6.                  learning_rate,
  7.                  weight_decay_rate=0.0,
  8.                  beta_1=0.9,
  9.                  beta_2=0.999,
  10.                  epsilon=1e-6,
  11.                  exclude_from_weight_decay=None,
  12.                  name="AdamWeightDecayOptimizer"):
  13.         """Constructs a AdamWeightDecayOptimizer."""
  14.         super(AdamWeightDecayOptimizer, self).__init__(False, name)
  15.  
  16.         self.learning_rate = learning_rate
  17.         self.weight_decay_rate = weight_decay_rate
  18.         self.beta_1 = beta_1
  19.         self.beta_2 = beta_2
  20.         self.epsilon = epsilon
  21.         self.exclude_from_weight_decay = exclude_from_weight_decay
  22.  
  23.     def apply_gradients(self, grads_and_vars, global_step=None, name=None):
  24.         """See base class."""
  25.         assignments = []
  26.         for (grad, param) in grads_and_vars:
  27.             if grad is None or param is None:
  28.                 continue
  29.  
  30.             param_name = self._get_variable_name(param.name)
  31.  
  32.             m = tf.get_variable(
  33.                 name=param_name + "/adam_m",
  34.                 shape=param.shape.as_list(),
  35.                 dtype=tf.float32,
  36.                 trainable=False,
  37.                 initializer=tf.zeros_initializer())
  38.             v = tf.get_variable(
  39.                 name=param_name + "/adam_v",
  40.                 shape=param.shape.as_list(),
  41.                 dtype=tf.float32,
  42.                 trainable=False,
  43.                 initializer=tf.zeros_initializer())
  44.  
  45.             # Standard Adam update.
  46.             next_m = (
  47.                     tf.multiply(self.beta_1, m) + tf.multiply(1.0 - self.beta_1, grad))
  48.             next_v = (
  49.                     tf.multiply(self.beta_2, v) + tf.multiply(1.0 - self.beta_2,
  50.                                                               tf.square(grad)))
  51.  
  52.             update = next_m / (tf.sqrt(next_v) + self.epsilon)
  53.  
  54.             # Just adding the square of the weights to the loss function is *not*
  55.             # the correct way of using L2 regularization/weight decay with Adam,
  56.             # since that will interact with the m and v parameters in strange ways.
  57.             #
  58.             # Instead we want ot decay the weights in a manner that doesn't interact
  59.             # with the m/v parameters. This is equivalent to adding the square
  60.             # of the weights to the loss with plain (non-momentum) SGD.
  61.             if self._do_use_weight_decay(param_name):
  62.                 update += self.weight_decay_rate * param
  63.  
  64.             update_with_lr = self.learning_rate * update
  65.  
  66.             next_param = param - update_with_lr
  67.  
  68.             assignments.extend(
  69.                 [param.assign(next_param),
  70.                  m.assign(next_m),
  71.                  v.assign(next_v)])
  72.         return tf.group(*assignments, name=name)
  73.  
  74.     def _do_use_weight_decay(self, param_name):
  75.         """Whether to use L2 weight decay for `param_name`."""
  76.         if not self.weight_decay_rate:
  77.             return False
  78.         if self.exclude_from_weight_decay:
  79.             for r in self.exclude_from_weight_decay:
  80.                 if re.search(r, param_name) is not None:
  81.                     return False
  82.         return True
  83.  
  84.     def _get_variable_name(self, param_name):
  85.         """Get the variable name from the tensor name."""
  86.         m = re.match("^(.*):\\d+$", param_name)
  87.         if m is not None:
  88.             param_name = m.group(1)
  89.         return param_name
  90.  
  91.  
  92. # adam原始论文对应的源码
  93. # class AdamWeightDecayOptimizer(tf.train.Optimizer):
  94. #     """A basic Adam optimizer that includes "correct" L2 weight decay."""
  95.  
  96. #     def __init__(self,
  97. #                  learning_rate,
  98. #                  weight_decay_rate=0.0,
  99. #                  beta_1=0.9,
  100. #                  beta_2=0.999,
  101. #                  epsilon=1e-6,
  102. #                  exclude_from_weight_decay=None,
  103. #                  name="AdamWeightDecayOptimizer"):
  104. #         """Constructs a AdamWeightDecayOptimizer."""
  105. #         super(AdamWeightDecayOptimizer, self).__init__(False, name)
  106.  
  107. #         self.learning_rate = learning_rate
  108. #         self.weight_decay_rate = weight_decay_rate
  109. #         self.beta_1 = beta_1
  110. #         self.beta_2 = beta_2
  111. #         self.epsilon = epsilon
  112. #         self.exclude_from_weight_decay = exclude_from_weight_decay
  113. #         self.learning_rate_t = None
  114. #         self._beta1_t = None
  115. #         self._beta2_t = None
  116. #         self._epsilon_t = None
  117.     
  118. #     def _get_beta_accumulators(self):
  119. #         with ops.init_scope():
  120. #             if tf.executing_eagerly():
  121. #                 graph = None
  122. #             else:
  123. #                 graph = ops.get_default_graph()
  124. #             return (self._get_non_slot_variable("beta1_power", graph=graph),
  125. #                     self._get_non_slot_variable("beta2_power", graph=graph))
  126.  
  127.  
  128. #     def _prepare(self):
  129. #         self.learning_rate_t = ops.convert_to_tensor(
  130. #             self.learning_rate, name='learning_rate')
  131. #         self.weight_decay_rate_t = ops.convert_to_tensor(
  132. #             self.weight_decay_rate, name='weight_decay_rate')
  133. #         self.beta_1_t = ops.convert_to_tensor(self.beta_1, name='beta_1')
  134. #         self.beta_2_t = ops.convert_to_tensor(self.beta_2, name='beta_2')
  135. #         self.epsilon_t = ops.convert_to_tensor(self.epsilon, name='epsilon')
  136.  
  137. #     def _create_slots(self, var_list):
  138. #         first_var = min(var_list, key=lambda x: x.name)
  139. #         self._create_non_slot_variable(initial_value=self.beta_1,
  140. #                                     name="beta1_power",
  141. #                                     colocate_with=first_var)
  142. #         self._create_non_slot_variable(initial_value=self.beta_2,
  143. #                                     name="beta2_power",
  144. #                                     colocate_with=first_var)
  145. #         for v in var_list:
  146. #             self._zeros_slot(v, 'm', self._name)
  147. #             self._zeros_slot(v, 'v', self._name)
  148.  
  149. #     def _apply_dense(self, grad, var):
  150. #         learning_rate_t = math_ops.cast(
  151. #             self.learning_rate_t, var.dtype.base_dtype)
  152. #         beta_1_t = math_ops.cast(self.beta_1_t, var.dtype.base_dtype)
  153. #         beta_2_t = math_ops.cast(self.beta_2_t, var.dtype.base_dtype)
  154. #         epsilon_t = math_ops.cast(self.epsilon_t, var.dtype.base_dtype)
  155. #         weight_decay_rate_t = math_ops.cast(
  156. #             self.weight_decay_rate_t, var.dtype.base_dtype)
  157.  
  158. #         m = self.get_slot(var, 'm')
  159. #         v = self.get_slot(var, 'v')
  160. #         beta1_power, beta2_power = self._get_beta_accumulators()
  161. #         beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype)
  162. #         beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype)
  163. #         learning_rate_t = math_ops.cast(self.learning_rate_t, var.dtype.base_dtype)
  164. #         learning_rate_t = (learning_rate_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
  165.     
  166. #         # Standard Adam update.
  167. #         next_m = (
  168. #             tf.multiply(beta_1_t, m) +
  169. #             tf.multiply(1.0 - beta_1_t, grad))
  170. #         next_v = (
  171. #             tf.multiply(beta_2_t, v) + tf.multiply(1.0 - beta_2_t,
  172. #                                                    tf.square(grad)))
  173.  
  174. #         update = next_m / (tf.sqrt(next_v) + epsilon_t)
  175.  
  176. #         if self._do_use_weight_decay(var.name):
  177. #             update += weight_decay_rate_t * var
  178.  
  179. #         update_with_lr = learning_rate_t * update
  180.  
  181. #         next_param = var - update_with_lr
  182.  
  183. #         return control_flow_ops.group(*[var.assign(next_param),
  184. #                                         m.assign(next_m),
  185. #                                         v.assign(next_v)])
  186.  
  187. #     def _resource_apply_dense(self, grad, var):
  188. #         learning_rate_t = math_ops.cast(
  189. #             self.learning_rate_t, var.dtype.base_dtype)
  190. #         beta_1_t = math_ops.cast(self.beta_1_t, var.dtype.base_dtype)
  191. #         beta_2_t = math_ops.cast(self.beta_2_t, var.dtype.base_dtype)
  192. #         epsilon_t = math_ops.cast(self.epsilon_t, var.dtype.base_dtype)
  193. #         weight_decay_rate_t = math_ops.cast(
  194. #             self.weight_decay_rate_t, var.dtype.base_dtype)
  195.  
  196. #         m = self.get_slot(var, 'm')
  197. #         v = self.get_slot(var, 'v')
  198. #         beta1_power, beta2_power = self._get_beta_accumulators()
  199. #         beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype)
  200. #         beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype)
  201. #         learning_rate_t = math_ops.cast(self.learning_rate_t, var.dtype.base_dtype)
  202. #         learning_rate_t = (learning_rate_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
  203.     
  204. #         # Standard Adam update.
  205. #         next_m = (
  206. #             tf.multiply(beta_1_t, m) +
  207. #             tf.multiply(1.0 - beta_1_t, grad))
  208. #         next_v = (
  209. #             tf.multiply(beta_2_t, v) + tf.multiply(1.0 - beta_2_t,
  210. #                                                    tf.square(grad)))
  211.  
  212. #         update = next_m / (tf.sqrt(next_v) + epsilon_t)
  213.  
  214. #         if self._do_use_weight_decay(var.name):
  215. #             update += weight_decay_rate_t * var
  216.  
  217. #         update_with_lr = learning_rate_t * update
  218.  
  219. #         next_param = var - update_with_lr
  220.  
  221. #         return control_flow_ops.group(*[var.assign(next_param),
  222. #                                         m.assign(next_m),
  223. #                                         v.assign(next_v)])
  224.  
  225. #     def _apply_sparse_shared(self, grad, var, indices, scatter_add):
  226. #         learning_rate_t = math_ops.cast(
  227. #             self.learning_rate_t, var.dtype.base_dtype)
  228. #         beta_1_t = math_ops.cast(self.beta_1_t, var.dtype.base_dtype)
  229. #         beta_2_t = math_ops.cast(self.beta_2_t, var.dtype.base_dtype)
  230. #         epsilon_t = math_ops.cast(self.epsilon_t, var.dtype.base_dtype)
  231. #         weight_decay_rate_t = math_ops.cast(
  232. #             self.weight_decay_rate_t, var.dtype.base_dtype)
  233.  
  234. #         m = self.get_slot(var, 'm')
  235. #         v = self.get_slot(var, 'v')
  236. #         beta1_power, beta2_power = self._get_beta_accumulators()
  237. #         beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype)
  238. #         beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype)
  239. #         learning_rate_t = math_ops.cast(self.learning_rate_t, var.dtype.base_dtype)
  240. #         learning_rate_t = (learning_rate_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
  241.     
  242. #         m_t = state_ops.assign(m, m * beta_1_t,
  243. #                                use_locking=self._use_locking)
  244.  
  245. #         m_scaled_g_values = grad * (1 - beta_1_t)
  246. #         with ops.control_dependencies([m_t]):
  247. #             m_t = scatter_add(m, indices, m_scaled_g_values)
  248.  
  249. #         v_scaled_g_values = (grad * grad) * (1 - beta_2_t)
  250. #         v_t = state_ops.assign(v, v * beta_2_t, use_locking=self._use_locking)
  251. #         with ops.control_dependencies([v_t]):
  252. #             v_t = scatter_add(v, indices, v_scaled_g_values)
  253.  
  254. #         update = m_t / (math_ops.sqrt(v_t) + epsilon_t)
  255.  
  256. #         if self._do_use_weight_decay(var.name):
  257. #             update += weight_decay_rate_t * var
  258.  
  259. #         update_with_lr = learning_rate_t * update
  260.  
  261. #         var_update = state_ops.assign_sub(var,
  262. #                                           update_with_lr,
  263. #                                           use_locking=self._use_locking)
  264. #         return control_flow_ops.group(*[var_update, m_t, v_t])
  265.  
  266. #     def _apply_sparse(self, grad, var):
  267. #         return self._apply_sparse_shared(
  268. #             grad.values, var, grad.indices,
  269. #             lambda x, i, v: state_ops.scatter_add(  # pylint: disable=g-long-lambda
  270. #                 x, i, v, use_locking=self._use_locking))
  271.  
  272. #     def _resource_scatter_add(self, x, i, v):
  273. #         with ops.control_dependencies(
  274. #                 [resource_variable_ops.resource_scatter_add(
  275. #                     x.handle, i, v)]):
  276. #             return x.value()
  277.  
  278. #     def _resource_apply_sparse(self, grad, var, indices):
  279. #         return self._apply_sparse_shared(
  280. #             grad, var, indices, self._resource_scatter_add)
  281.  
  282. #     def _do_use_weight_decay(self, param_name):
  283. #         """Whether to use L2 weight decay for `param_name`."""
  284. #         if not self.weight_decay_rate:
  285. #             return False
  286. #         if self.exclude_from_weight_decay:
  287. #             for r in self.exclude_from_weight_decay:
  288. #                 if re.search(r, param_name) is not None:
  289. #                     return False
  290. #         return True
  291. #     def _finish(self, update_ops, name_scope):
  292. #         # Update the power accumulators.
  293. #         with ops.control_dependencies(update_ops):
  294. #             beta1_power, beta2_power = self._get_beta_accumulators()
  295. #             with ops.colocate_with(beta1_power):
  296. #                 update_beta1 = beta1_power.assign(
  297. #                     beta1_power * self.beta_1_t, use_locking=self._use_locking)
  298. #                 update_beta2 = beta2_power.assign(
  299. #                     beta2_power * self.beta_2_t, use_locking=self._use_locking)
  300. #             return control_flow_ops.group(*update_ops + [update_beta1, update_beta2],
  301. #                                         name=name_scope)

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/天景科技苑/article/detail/946054
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号