diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7444713 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +dist/ +*.egg-info/ \ No newline at end of file diff --git a/plugins/gelu.py b/plugins/gelu.py deleted file mode 100644 index 2a907ed..0000000 --- a/plugins/gelu.py +++ /dev/null @@ -1,15 +0,0 @@ -from leanpass_plugins import PluginInfo, op -import numpy as np - -info = PluginInfo( - name="gelu", - author="Test Contributor", - github_username="testcontrib", - description="Gaussian Error Linear Unit activation function." -) - -@op(info.name) -def gelu_forward(x): - # Standard GELU approximation - return 0.5 * x * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3)))) - diff --git a/plugins/mish.py b/plugins/mish.py new file mode 100644 index 0000000..be9e5a1 --- /dev/null +++ b/plugins/mish.py @@ -0,0 +1,17 @@ +from leanpass_plugins import PluginInfo, op +import numpy as np + +info = PluginInfo( + name="mish", + author="Test Contributor", + github_username="testcontrib", + description="Mish activation function: x * tanh(softplus(x))." +) + +@op(info.name) +def mish_forward(x): + # Mish: x * tanh(ln(1 + e^x)) + # np.log1p(np.exp(x)) is a numerically stable softplus + softplus_x = np.log1p(np.exp(x)) + return x * np.tanh(softplus_x) +