Staging PR for implimenting Phi-2 support.

experiments/benchmark.ipynb

@@ -0,0 +1,32 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "unsloth",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

tests/kernels/conftest.py

@@ -0,0 +1,73 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import gc
+from contextlib import contextmanager
+
+import os
+import pytest
+import numpy as np
+import torch
+import torch._dynamo as dynamo
+
+
+@contextmanager
+def set_seed(seed: int = 0):
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.backends.cudnn.deterministic = True
+    torch.backends.cudnn.benchmark = False
+    torch.use_deterministic_algorithms(True)
+    os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8"
+    yield
+
+@pytest.fixture(autouse=True)
+def reset_dyno_state():
+    cache_limit = dynamo.config.cache_size_limit
+    try:
+        dynamo.config.cache_size_limit = 8192
+        dynamo.reset()
+        torch.cuda.synchronize()
+        gc.collect()
+        torch.cuda.empty_cache()
+        yield {}
+    except RuntimeError as err:
+        raise err
+    finally:
+        dynamo.config.cache_size_limit = cache_limit
+        torch.cuda.synchronize()
+        gc.collect()
+        torch.cuda.empty_cache()
+
+
+def assert_all_close(a: torch.Tensor, b: torch.Tensor, rtol=0, atol=1e-1) -> None:
+    """
+    Check that all elements of tensors a and b are within provided thresholds.
+    """
+    assert a.shape == b.shape, f"Shapes don't match: {a.shape} != {b.shape}"
+    assert a.dtype == b.dtype, f"Dtypes don't match: {a.dtype} != {b.dtype}"
+    assert a.device == b.device, f"Devices don't match: {a.device} != {b.device}"
+    max_abs_diff = torch.max(torch.abs(a - b))
+    rel_diff = torch.abs(a / b)
+    max_rel_diff = torch.max(rel_diff)
+    mismatch_elements = torch.sum(torch.abs(a - b) > atol + rtol * torch.abs(b))
+    nb_elements = torch.numel(a)
+    msg = (
+        f"Differences: "
+        f"{max_abs_diff:.3f} (max abs), "
+        f"{max_rel_diff:.3f} (max rel), "
+        f"{mismatch_elements}/{nb_elements} (mismatch elements)"
+    )
+    assert torch.allclose(a, b, rtol=rtol, atol=atol), msg

tests/kernels/test_crossentropy.py

@@ -0,0 +1,22 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import torch 
+from unsloth.kernels.cross_entropy_loss import fast_cross_entropy_loss
+
+x = torch.randn(1, 126, 51200, device='cuda')
+y = torch.randn(1, 126, device='cuda')
+
+fast_cross_entropy_loss(logits=x,labels=y)

tests/kernels/test_gelu.py

@@ -0,0 +1,55 @@
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import pytest
+import torch
+
+from unsloth.kernels.gelu import gelu_forward_kenel, gelu_backward_kernel
+from tests.kernels.conftest import set_seed, assert_all_close
+
+@set_seed
+@pytest.fixture(params=[(100, 100), (1024, 1024), (5000, 1024), (12345, 5678)])
+def test_matrix(request):
+    shape = request.param
+    x = torch.randn(shape, device='cuda')
+    return x
+
+# Test function
+def test_relu_kernel_fwd(test_matrix):
+    # Apply your Triton-based ReLU kernel
+    triton_output = gelu_forward_kenel(test_matrix)
+
+    # Apply PyTorch's ReLU for comparison
+    torch_gelu = torch.nn.GELU()
+    torch_output = torch_gelu(test_matrix)
+
+    # Check if the outputs are close enough using assert_all_close
+    assert_all_close(triton_output, torch_output, rtol=1e-05, atol=1e-08)
+
+
+# Test function for GeLU backward kernel
+def test_gelu_backward_kernel(test_matrix):
+    # Create a tensor representing gradients (e.g., random gradients)
+    grad_input = torch.randn_like(test_matrix)
+
+    # Apply your Triton-based GeLU backward kernel
+    triton_output = gelu_backward_kernel(test_matrix, grad_input)
+
+    # Compute PyTorch's GeLU gradient for comparison
+    torch_gelu = torch.nn.GELU()
+    torch_output = torch.autograd.grad(torch_output.sum(), test_matrix, grad_outputs=grad_input)[0]
+
+    # Check if the outputs are close enough using assert_all_close
+    assert_all_close(triton_output, torch_output, rtol=1e-05, atol=1e-08)