File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -117,7 +117,7 @@ def loss_ownership_corr(
117117 # So we need to multiply vectors x by a factor of tanh(|x|) / |x| = tanh(sqrt(|x|^2)) / sqrt(|x|^2)
118118 # But there's a division by 0 when |x| = 0, and also sqrt(0) has infinite gradient..
119119 # So to do this in a numerically stable way, we do this piecewise, using 3rd order taylor expansion
120- # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/105 x^3.
120+ # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/315 x^3.
121121 delta = 0.010
122122 sqrtmagsqboundedbelow = torch .sqrt (torch .clamp (magsq ,min = 0.008 ))
123123 magsqboundedabove = torch .clamp (magsq ,max = 0.012 )
@@ -159,7 +159,7 @@ def loss_futurepos_corr(
159159 # So we need to multiply vectors x by a factor of tanh(|x|) / |x| = tanh(sqrt(|x|^2)) / sqrt(|x|^2)
160160 # But there's a division by 0 when |x| = 0, and also sqrt(0) has infinite gradient..
161161 # So to do this in a numerically stable way, we do this piecewise, using 3rd order taylor expansion
162- # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/105 x^3.
162+ # around 0. Taylor expansion of tanh(sqrt(x)) / sqrt(x) is 1 - 1/3 x + 2/15 x^2 - 17/315 x^3.
163163 delta = 0.010
164164 sqrtmagsqboundedbelow = torch .sqrt (torch .clamp (magsq ,min = 0.008 ))
165165 magsqboundedabove = torch .clamp (magsq ,max = 0.012 )
You can’t perform that action at this time.
0 commit comments