Rewritten attention system & Added basic embedding encoding

xEcho1337 · Dec 2, 2024 · e04e7ba · e04e7ba
1 parent cb08e2f
commit e04e7ba
Show file tree

Hide file tree

Showing 22 changed files with 320 additions and 1,586 deletions.
diff --git a/src/main/java/net/echo/brain4j/activation/Activation.java b/src/main/java/net/echo/brain4j/activation/Activation.java
@@ -7,8 +7,10 @@
 public interface Activation {
 
     double activate(double input);
+
     double[] activate(double[] input);
 
     double getDerivative(double input);
+
     void apply(List<Neuron> neurons);
 }
diff --git a/src/main/java/net/echo/brain4j/activation/impl/SoftmaxActivation.java b/src/main/java/net/echo/brain4j/activation/impl/SoftmaxActivation.java
@@ -6,6 +6,7 @@
 import java.util.List;
 
 public class SoftmaxActivation implements Activation {
+
     @Override
     public double activate(double input) {
         throw new UnsupportedOperationException("Softmax activation function is not supported for single value");
@@ -14,6 +15,7 @@ public double activate(double input) {
     @Override
     public double[] activate(double[] inputs) {
         double maxInput = Double.NEGATIVE_INFINITY;
+
         for (double input : inputs) {
             if (input > maxInput) {
                 maxInput = input;
@@ -22,13 +24,13 @@ public double[] activate(double[] inputs) {
 
         double[] expValues = new double[inputs.length];
         double sum = 0.0;
+
         for (int i = 0; i < inputs.length; i++) {
-            expValues[i] = Math.exp(inputs[i] - maxInput);
-            sum += expValues[i];
+            sum += (expValues[i] = Math.exp(inputs[i] - maxInput));
         }
 
         for (int i = 0; i < expValues.length; i++) {
-            expValues[i] /= sum;
+            expValues[i] = expValues[i] / sum;
         }
 
         return expValues;

diff --git a/src/main/java/net/echo/brain4j/nlp/AlphabetInitialization.java b/src/main/java/net/echo/brain4j/nlp/AlphabetInitialization.java
diff --git a/src/main/java/net/echo/brain4j/nlp/LabelTransformer.java b/src/main/java/net/echo/brain4j/nlp/LabelTransformer.java
diff --git a/src/main/java/net/echo/brain4j/nlp/agents/Agent.java b/src/main/java/net/echo/brain4j/nlp/agents/Agent.java
diff --git a/src/main/java/net/echo/brain4j/nlp/agents/attention/AttentionMechanism.java b/src/main/java/net/echo/brain4j/nlp/agents/attention/AttentionMechanism.java
diff --git a/src/main/java/net/echo/brain4j/nlp/agents/attention/impl/MultiHeadAttention.java b/src/main/java/net/echo/brain4j/nlp/agents/attention/impl/MultiHeadAttention.java
diff --git a/src/main/java/net/echo/brain4j/nlp/agents/attention/impl/SelfAttention.java b/src/main/java/net/echo/brain4j/nlp/agents/attention/impl/SelfAttention.java
diff --git a/src/main/java/net/echo/brain4j/nlp/agents/attention/score/AttentionScorer.java b/src/main/java/net/echo/brain4j/nlp/agents/attention/score/AttentionScorer.java