more activations

activation/elu.go

@@ -0,0 +1,25 @@
+package activation
+
+import (
+	"math"
+
+	"github.com/julioguillermo/godeep/types"
+)
+
+type ELU[T types.Number] struct {
+	Alpha T
+}
+
+func (p ELU[T]) Activate(n T) T {
+	if n >= 0 {
+		return n
+	}
+	return p.Alpha * (T(math.Exp(float64(n)) - 1))
+}
+
+func (p ELU[T]) Derive(n T) T {
+	if n >= 0 {
+		return n
+	}
+	return p.Activate(n) + p.Alpha
+}

activation/gelu.go

@@ -0,0 +1,23 @@
+package activation
+
+import (
+	"math"
+
+	"github.com/julioguillermo/godeep/types"
+)
+
+type GELU[T types.Number] struct{}
+
+func (GELU[T]) Activate(n T) T {
+	x := float64(n)
+	return T(0.5 * x * (1 + math.Tanh(math.Sqrt(2/math.Pi)*(x+0.044715*math.Pow(x, 3)))))
+}
+
+func (p GELU[T]) Derive(x T) T {
+	h := 0.000001
+	H := T(h)
+	if H == 0 {
+		H++
+	}
+	return (p.Activate(x+H) - p.Activate(x-H)) / (2 * H)
+}

activation/relu.go

@@ -5,19 +5,19 @@ import (
 )
 
 type Relu[T types.Number] struct {
-	M T
+	Alpha T
 }
 
 func (p Relu[T]) Activate(t T) T {
 	if t < 0 {
-		return t * p.M
+		return t * p.Alpha
 	}
 	return t
 }
 
 func (p Relu[T]) Derive(t T) T {
 	if t < 0 {
-		return p.M
+		return p.Alpha
 	}
 	return 1
 }

activation/selu.go

@@ -0,0 +1,18 @@
+package activation
+
+import (
+	"github.com/julioguillermo/godeep/types"
+)
+
+type SELU[T types.Number] struct {
+	ELU[T]
+	Lambda T
+}
+
+func (p SELU[T]) Activate(n T) T {
+	return p.Lambda * p.ELU.Activate(n)
+}
+
+func (p SELU[T]) Derive(n T) T {
+	return p.Lambda * p.ELU.Derive(n)
+}

activation/swish.go

@@ -0,0 +1,17 @@
+package activation
+
+import (
+	"github.com/julioguillermo/godeep/types"
+)
+
+type Swish[T types.Number] struct {
+	Sigmoid[T]
+}
+
+func (p Swish[T]) Activate(t T) T {
+	return t * p.Sigmoid.Activate(t)
+}
+
+func (p Swish[T]) Derive(t T) T {
+	return p.Activate(t) + p.Sigmoid.Activate(t)*(1-p.Activate(t))
+}