Merge pull request #387 from apphp/ML-385-convert-cost-functions-to-NumPower

Ml 385 convert cost functions to num power

Author: apphp

docs/neural-network/cost-functions/cross-entropy.md

@@ -12,7 +12,7 @@ This cost function does not have any parameters.
 
 ## Example
 ```php
-use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy;
+use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy\CrossEntropy;
 
 $costFunction = new CrossEntropy();
-```
+```

docs/neural-network/cost-functions/huber-loss.md

@@ -5,10 +5,10 @@ The pseudo Huber Loss function transitions between L1 and L2 loss at a given piv
 
 $$
 L_{\delta}=
-    \left\{\begin{matrix}
-        \frac{1}{2}(y - \hat{y})^{2} & if \left | (y - \hat{y})  \right | < \delta\\
-        \delta ((y - \hat{y}) - \frac1 2 \delta) & otherwise
-    \end{matrix}\right.
+    \begin{cases}
+        \frac{1}{2}(y - \hat{y})^{2} & \text{if } |y - \hat{y}| < \delta\\
+        \delta ((y - \hat{y}) - \frac{1}{2} \delta) & \text{otherwise}
+    \end{cases}
 $$
 
 ## Parameters
@@ -21,4 +21,4 @@ $$
 use Rubix\ML\NeuralNet\CostFunctions\HuberLoss;
 
 $costFunction = new HuberLoss(0.5);
-```
+```

docs/neural-network/cost-functions/least-squares.md

@@ -12,7 +12,7 @@ This cost function does not have any parameters.
 
 ## Example
 ```php
-use Rubix\ML\NeuralNet\CostFunctions\LeastSquares;
+use Rubix\ML\NeuralNet\CostFunctions\LeastSquares\LeastSquares;
 
 $costFunction = new LeastSquares();
-```
+```

docs/neural-network/cost-functions/mean-absolute-error.md

@@ -0,0 +1,18 @@
+<span style="float:right;"><a href="https://github.com/RubixML/ML/blob/master/src/NeuralNet/CostFunctions/MeanAbsoluteError/MeanAbsoluteError.php">[source]</a></span>
+
+# Mean Absolute Error
+Mean Absolute Error (MAE) measures the average magnitude of errors between predicted and actual values without considering their direction. It is a linear score which means all individual differences are weighted equally. MAE is more robust to outliers compared to Mean Squared Error (MSE) because it doesn't square the differences.
+
+$$
+MAE = \frac{1}{n}\sum_{i=1}^{n}|y_i - \hat{y}_i|
+$$
+
+## Parameters
+This cost function does not have any parameters.
+
+## Example
+```php
+use Rubix\ML\NeuralNet\CostFunctions\MeanAbsoluteError\MeanAbsoluteError;
+
+$costFunction = new MeanAbsoluteError();
+```

docs/neural-network/cost-functions/relative-entropy.md

@@ -12,7 +12,7 @@ This cost function does not have any parameters.
 
 ## Example
 ```php
-use Rubix\ML\NeuralNet\CostFunctions\RelativeEntropy;
+use Rubix\ML\NeuralNet\CostFunctions\RelativeEntropy\RelativeEntropy;
 
 $costFunction = new RelativeEntropy();
-```
+```

src/NeuralNet/ActivationFunctions/GELU/GELU.php

@@ -33,7 +33,10 @@ class GELU implements ActivationFunction, IBufferDerivative
      * @var float
      */
     protected const ALPHA = 0.7978845608;
-    /** @var float 0.5 * ALPHA */
+
+    /**
+     * @var float 0.5 * ALPHA
+     */
     protected const HALF_ALPHA = 0.3989422804;
 
     /**
@@ -42,7 +45,10 @@ class GELU implements ActivationFunction, IBufferDerivative
      * @var float
      */
     protected const BETA = 0.044715;
-    /** @var float 3 * BETA */
+
+    /**
+     * @var float 3 * BETA
+     */
     protected const TRIPLE_BETA = 0.134145;
 
     /**

src/NeuralNet/CostFunctions/Base/Contracts/ClassificationLoss.php

@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rubix\ML\NeuralNet\CostFunctions\Base\Contracts;
+
+interface ClassificationLoss extends CostFunction
+{
+    //
+}

src/NeuralNet/CostFunctions/Base/Contracts/CostFunction.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rubix\ML\NeuralNet\CostFunctions\Base\Contracts;
+
+use NDArray;
+use Stringable;
+
+/**
+ * Cost Function
+ *
+ * @category    Machine Learning
+ * @package     Rubix/ML
+ * @author      Samuel Akopyan <[email protected]>
+ */
+interface CostFunction extends Stringable
+{
+    /**
+     * Compute the loss score.
+     *
+     * @param NDArray $output
+     * @param NDArray $target
+     * @return float
+     */
+    public function compute(NDArray $output, NDArray $target) : float;
+
+    /**
+     * Calculate the gradient of the cost function with respect to the output.
+     *
+     * @param NDArray $output
+     * @param NDArray $target
+     * @return NDArray
+     */
+    public function differentiate(NDArray $output, NDArray $target) : NDArray;
+}

src/NeuralNet/CostFunctions/Base/Contracts/RegressionLoss.php

@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rubix\ML\NeuralNet\CostFunctions\Base\Contracts;
+
+interface RegressionLoss extends CostFunction
+{
+    //
+}

src/NeuralNet/CostFunctions/CrossEntropy/CrossEntropy.php

@@ -0,0 +1,91 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Rubix\ML\NeuralNet\CostFunctions\CrossEntropy;
+
+use NDArray;
+use NumPower;
+use Rubix\ML\NeuralNet\CostFunctions\Base\Contracts\ClassificationLoss;
+use Rubix\ML\Traits\AssertsShapes;
+use const Rubix\ML\EPSILON;
+
+/**
+ * Cross Entropy
+ *
+ * Cross Entropy, or log loss, measures the performance of a classification model
+ * whose output is a probability value between 0 and 1. Cross-entropy loss
+ * increases as the predicted probability diverges from the actual label. So
+ * predicting a probability of .012 when the actual observation label is 1 would
+ * be bad and result in a high loss value. A perfect score would have a log loss
+ * of 0.
+ *
+ * @category    Machine Learning
+ * @package     Rubix/ML
+ * @author      Andrew DalPino
+ * @author      Samuel Akopyan <[email protected]>
+ */
+class CrossEntropy implements ClassificationLoss
+{
+    use AssertsShapes;
+
+    /**
+     * Compute the loss score.
+     *
+     * L(y, ŷ) = -Σ(y * log(ŷ)) / n
+     *
+     * @param NDArray $output The output of the network
+     * @param NDArray $target The target values
+     * @return float
+     */
+    public function compute(NDArray $output, NDArray $target) : float
+    {
+        $this->assertSameShape($output, $target);
+
+        // Clip values to avoid log(0)
+        $output = NumPower::clip($output, EPSILON, 1.0);
+
+        $logOutput = NumPower::log($output);
+        $product = NumPower::multiply($target, $logOutput);
+        $negated = NumPower::multiply($product, -1.0);
+
+        return NumPower::mean($negated);
+    }
+
+    /**
+     * Calculate the gradient of the cost function with respect to the output.
+     *
+     * ∂L/∂ŷ = (ŷ - y) / (ŷ * (1 - ŷ))
+     *
+     * @param NDArray $output The output of the network
+     * @param NDArray $target The target values
+     * @return NDArray
+     */
+    public function differentiate(NDArray $output, NDArray $target) : NDArray
+    {
+        $this->assertSameShape($output, $target);
+
+        // Numerator = ŷ - y (calculate before clipping to preserve zeros)
+        $numerator = NumPower::subtract($output, $target);
+
+        // Clip values to avoid division by zero
+        $output = NumPower::clip($output, EPSILON, 1.0 - EPSILON);
+
+        // Denominator = ŷ * (1 - ŷ)
+        $oneMinusOutput = NumPower::subtract(1.0, $output);
+        $denominator = NumPower::multiply($output, $oneMinusOutput);
+        $denominator = NumPower::clip($denominator, EPSILON, 1.0);
+
+        return NumPower::divide($numerator, $denominator);
+    }
+
+    /**
+     * Return the string representation of the object.
+     *
+     * @return string
+     */
+    public function __toString() : string
+    {
+        return 'Cross Entropy';
+    }
+}