Added hard sigmoid function

src/NeuralNet/ActivationFunctions/HardSigmoid/HardSigmoid.php

@@ -1,62 +1,109 @@
 <?php
 
-namespace Rubix\ML\NeuralNet\ActivationFunctions;
+declare(strict_types=1);
 
-use Tensor\Matrix;
+namespace Rubix\ML\NeuralNet\ActivationFunctions\HardSigmoid;
+
+use NumPower;
+use NDArray;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\ActivationFunction;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IBufferDerivative;
 
 /**
- * Sigmoid
+ * HardSigmoid
+ *
+ * A piecewise linear approximation of the sigmoid function that is computationally
+ * more efficient. The Hard Sigmoid function has an output value between 0 and 1,
+ * making it useful for binary classification problems.
  *
- * A bounded S-shaped function (sometimes called the *Logistic* function) with an output value
- * between 0 and 1. The output of the sigmoid function has the advantage of being interpretable
- * as a probability, however it is not zero-centered and tends to saturate if inputs become large.
+ * f(x) = max(0, min(1, 0.2 * x + 0.5))
  *
  * @category    Machine Learning
  * @package     Rubix/ML
  * @author      Andrew DalPino
+ * @author      Samuel Akopyan <[email protected]>
  */
-class HardSigmoid implements ActivationFunction
+class HardSigmoid implements ActivationFunction, IBufferDerivative
 {
     /**
-     * Compute the activation.
+     * The slope of the linear region.
+     *
+     * @var float
+     */
+    protected const SLOPE = 0.2;
+
+    /**
+     * The y-intercept of the linear region.
+     *
+     * @var float
+     */
+    protected const INTERCEPT = 0.5;
+
+    /**
+     * The lower bound of the linear region.
      *
-     * @internal
+     * @var float
+     */
+    protected const LOWER_BOUND = -2.5;
+
+    /**
+     * The upper bound of the linear region.
+     *
+     * @var float
+     */
+    protected const UPPER_BOUND = 2.5;
+
+    /**
+     * Apply the HardSigmoid activation function to the input.
+     *
+     * f(x) = max(0, min(1, 0.2 * x + 0.5))
      *
-     * @param Matrix $input
-     * @return Matrix
+     * @param NDArray $input The input values
+     * @return NDArray The activated values
      */
-    public function activate(Matrix $input) : Matrix
+    public function activate(NDArray $input) : NDArray
     {
-        return NumPower::clip(0.2 * $input + 0.5, 0, 1);
+        // Calculate 0.2 * x + 0.5
+        $linear = NumPower::add(
+            NumPower::multiply(self::SLOPE, $input),
+            self::INTERCEPT
+        );
+
+        // Clip values between 0 and 1
+        return NumPower::clip($linear, 0.0, 1.0);
     }
 
     /**
-     * Calculate the derivative of the activation.
+     * Calculate the derivative of the activation function.
      *
-     * @internal
+     * f'(x) = 0.2 if -2.5 < x < 2.5
+     * f'(x) = 0   otherwise
      *
-     * @param Matrix $input
-     * @param Matrix $output
-     * @return Matrix
+     * @param NDArray $x Input matrix
+     * @return NDArray Derivative matrix
      */
-    public function differentiate(Matrix $input, Matrix $output) : Matrix
+    public function differentiate(NDArray $x) : NDArray
     {
-        $lowPart = NumPower::lessEqual($input, -2.5);
-        $highPart = NumPower::greaterEqual($input, 2.5);
-        $union = $lowPart + $highPart;
+        // For values in the linear region (-2.5 < x < 2.5): SLOPE
+        $inLinearRegion = NumPower::greater($x, self::LOWER_BOUND);
+        $inLinearRegion = NumPower::multiply($inLinearRegion, NumPower::less($x, self::UPPER_BOUND));
+        $linearPart = NumPower::multiply($inLinearRegion, self::SLOPE);
 
-        return NumPower::equal($union, 0) * 0.2;
+        // For values outside the linear region: 0
+        // Since we're multiplying by 0 for these regions, we don't need to explicitly handle them
+        // The mask $inLinearRegion already contains 0s for x <= -2.5 and x >= 2.5,
+        // so when we multiply by SLOPE, those values remain 0 in the result
+
+        return $linearPart;
     }
 
     /**
-     * Return the string representation of the object.
-     *
-     * @internal
+     * Return the string representation of the activation function.
      *
-     * @return string
+     * @return string String representation
      */
     public function __toString() : string
     {
-        return 'Sigmoid';
+        return 'HardSigmoid';
     }
 }

tests/NeuralNet/ActivationFunctions/HardSigmoid/HardSigmoidTest.php

@@ -0,0 +1,119 @@
+<?php
+
+declare(strict_types = 1);
+
+namespace Rubix\ML\Tests\NeuralNet\ActivationFunctions\HardSigmoid;
+
+use PHPUnit\Framework\Attributes\CoversClass;
+use PHPUnit\Framework\Attributes\DataProvider;
+use PHPUnit\Framework\Attributes\Group;
+use PHPUnit\Framework\Attributes\Test;
+use PHPUnit\Framework\Attributes\TestDox;
+use NumPower;
+use NDArray;
+use Rubix\ML\NeuralNet\ActivationFunctions\HardSigmoid\HardSigmoid;
+use PHPUnit\Framework\TestCase;
+use Generator;
+
+#[Group('ActivationFunctions')]
+#[CoversClass(HardSigmoid::class)]
+class HardSigmoidTest extends TestCase
+{
+    /**
+     * @var HardSigmoid
+     */
+    protected HardSigmoid $activationFn;
+
+    /**
+     * @return Generator<array>
+     */
+    public static function computeProvider() : Generator
+    {
+        yield [
+            NumPower::array([
+                [2.5, 2.4, 2.0, 1.0, -0.5, 0.0, 20.0, -2.5, -2.4, -10.0],
+            ]),
+            [
+                [1.0, 0.9800000190734863, 0.8999999761581421, 0.699999988079071, 0.4000000059604645, 0.5, 1.0, 0.0, 0.019999980926513672, 0.0],
+            ],
+        ];
+
+        yield [
+            NumPower::array([
+                [-0.12, 0.31, -0.49],
+                [0.99, 0.08, -0.03],
+                [0.05, -0.52, 0.54],
+            ]),
+            [
+                [0.47600001096725464, 0.5619999766349792, 0.4020000100135803],
+                [0.6980000138282776, 0.515999972820282, 0.49399998784065247],
+                [0.5099999904632568, 0.3959999978542328, 0.6079999804496765],
+            ],
+        ];
+    }
+
+    /**
+     * @return Generator<array>
+     */
+    public static function differentiateProvider() : Generator
+    {
+        yield [
+            NumPower::array([
+                [2.5, 1.0, -0.5, 0.0, 20.0, -10.0],
+            ]),
+            [
+                [0.0, 0.20000000298023224, 0.20000000298023224, 0.20000000298023224, 0.0, 0.0],
+            ],
+        ];
+
+        yield [
+            NumPower::array([
+                [-0.12, 0.31, -0.49],
+                [2.99, 0.08, -2.03],
+                [0.05, -0.52, 0.54],
+            ]),
+            [
+                [0.20000000298023224, 0.20000000298023224, 0.20000000298023224],
+                [0.0, 0.20000000298023224, 0.20000000298023224],
+                [0.20000000298023224, 0.20000000298023224, 0.20000000298023224],
+            ],
+        ];
+    }
+
+    /**
+     * Set up the test case.
+     */
+    protected function setUp() : void
+    {
+        parent::setUp();
+
+        $this->activationFn = new HardSigmoid();
+    }
+
+    #[Test]
+    #[TestDox('Can be cast to a string')]
+    public function testToString() : void
+    {
+        static::assertEquals('HardSigmoid', (string) $this->activationFn);
+    }
+
+    #[Test]
+    #[TestDox('Correctly activates the input')]
+    #[DataProvider('computeProvider')]
+    public function testActivate(NDArray $input, array $expected) : void
+    {
+        $activations = $this->activationFn->activate($input)->toArray();
+
+        static::assertEquals($expected, $activations);
+    }
+
+    #[Test]
+    #[TestDox('Correctly differentiates the input')]
+    #[DataProvider('differentiateProvider')]
+    public function testDifferentiate(NDArray $input, array $expected) : void
+    {
+        $derivatives = $this->activationFn->differentiate($input)->toArray();
+
+        static::assertEquals($expected, $derivatives);
+    }
+}