{ "cells": [ { "cell_type": "markdown", "id": "14ac6109", "metadata": { "id": "IqM-T1RTzY6C", "papermill": { "duration": 0.021668, "end_time": "2025-01-01T01:36:49.042245", "exception": false, "start_time": "2025-01-01T01:36:49.020577", "status": "completed" }, "tags": [] }, "source": [ "To run this, press \"*Runtime*\" and press \"*Run all*\" on a **free** Tesla T4 Google Colab instance!\n", "
\n", "\n", "To install Unsloth on your own computer, follow the installation instructions on our Github page [here](https://github.com/unslothai/unsloth#installation-instructions---conda).\n", "\n", "You will learn how to do [data prep](#Data), how to [train](#Train), how to [run the model](#Inference), & [how to save it](#Save) (eg for Llama.cpp)." ] }, { "cell_type": "markdown", "id": "992529ab", "metadata": { "papermill": { "duration": 0.017829, "end_time": "2025-01-01T01:36:49.078747", "exception": false, "start_time": "2025-01-01T01:36:49.060918", "status": "completed" }, "tags": [] }, "source": [ "## Kaggle is slow - you'll have to wait **5 minutes** for it to install.\n", "\n", "I suggest you to use our free Colab notebooks instead. I linked our Mistral Colab notebook here: [notebook](https://colab.research.google.com/drive/1Dyauq4kTZoLewQ1cApceUQVNcnnNTzg_?usp=sharing)" ] }, { "cell_type": "code", "execution_count": 1, "id": "5891519d", "metadata": { "execution": { "iopub.execute_input": "2025-01-01T01:36:49.115782Z", "iopub.status.busy": "2025-01-01T01:36:49.115557Z", "iopub.status.idle": "2025-01-01T01:39:57.034466Z", "shell.execute_reply": "2025-01-01T01:39:57.033580Z" }, "papermill": { "duration": 187.938919, "end_time": "2025-01-01T01:39:57.036077", "exception": false, "start_time": "2025-01-01T01:36:49.097158", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting pip3-autoremove\r\n", " Downloading pip3_autoremove-1.2.2-py2.py3-none-any.whl.metadata (2.2 kB)\r\n", "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (from pip3-autoremove) (24.1.2)\r\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from pip3-autoremove) (71.0.4)\r\n", "Downloading pip3_autoremove-1.2.2-py2.py3-none-any.whl (6.7 kB)\r\n", "Installing collected packages: pip3-autoremove\r\n", "Successfully installed pip3-autoremove-1.2.2\r\n", "pyarrow 18.1.0 is installed but pyarrow<15.0.0a0,>=14.0.1 is required\r\n", "Redoing requirement with just package name...\r\n", "dask 2024.12.1 is installed but dask==2024.8.0 is required\r\n", "Redoing requirement with just package name...\r\n", "google-api-core 1.34.1 is installed but google-api-core[grpc]<3.0.0dev,>=2.16.0 is required\r\n", "Redoing requirement with just package name...\r\n", "notebook 6.5.4 is installed but notebook==6.5.5 is required\r\n", "Redoing requirement with just package name...\r\n", "pyarrow 18.1.0 is installed but pyarrow<16,>=2 is required\r\n", "Redoing requirement with just package name...\r\n", "tbb 2022.0.0 is installed but tbb==2021.* is required\r\n", "Redoing requirement with just package name...\r\n", "google-api-core 1.34.1 is installed but google-api-core<3.0.0dev,>=2.10.2 is required\r\n", "Redoing requirement with just package name...\r\n", "The 'pycairo>=1.16.0' distribution was not found and is required by the application\r\n", "Skipping pycairo\r\n", "torchvision 0.19.1+cu121 (/usr/local/lib/python3.10/dist-packages)\r\n", " torch 2.4.1+cu121 (/usr/local/lib/python3.10/dist-packages)\r\n", " sympy 1.13.3 (/usr/local/lib/python3.10/dist-packages)\r\n", " mpmath 1.3.0 (/usr/local/lib/python3.10/dist-packages)\r\n", "torch 2.4.1+cu121 (/usr/local/lib/python3.10/dist-packages)\r\n", " sympy 1.13.3 (/usr/local/lib/python3.10/dist-packages)\r\n", " mpmath 1.3.0 (/usr/local/lib/python3.10/dist-packages)\r\n", "torchaudio 2.4.1+cu121 (/usr/local/lib/python3.10/dist-packages)\r\n", " torch 2.4.1+cu121 (/usr/local/lib/python3.10/dist-packages)\r\n", " sympy 1.13.3 (/usr/local/lib/python3.10/dist-packages)\r\n", " mpmath 1.3.0 (/usr/local/lib/python3.10/dist-packages)\r\n", "Found existing installation: sympy 1.13.3\r\n", "Uninstalling sympy-1.13.3:\r\n", " Successfully uninstalled sympy-1.13.3\r\n", "Found existing installation: torch 2.4.1+cu121\r\n", "Uninstalling torch-2.4.1+cu121:\r\n", " Successfully uninstalled torch-2.4.1+cu121\r\n", "Found existing installation: mpmath 1.3.0\r\n", "Uninstalling mpmath-1.3.0:\r\n", " Successfully uninstalled mpmath-1.3.0\r\n", "Found existing installation: torchvision 0.19.1+cu121\r\n", "Uninstalling torchvision-0.19.1+cu121:\r\n", " Successfully uninstalled torchvision-0.19.1+cu121\r\n", "Found existing installation: torchaudio 2.4.1+cu121\r\n", "Uninstalling torchaudio-2.4.1+cu121:\r\n", " Successfully uninstalled torchaudio-2.4.1+cu121\r\n", "Looking in indexes: https://download.pytorch.org/whl/cu121\r\n", "Collecting torch\r\n", " Downloading https://download.pytorch.org/whl/cu121/torch-2.5.1%2Bcu121-cp310-cp310-linux_x86_64.whl (780.4 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m780.4/780.4 MB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting torchvision\r\n", " Downloading https://download.pytorch.org/whl/cu121/torchvision-0.20.1%2Bcu121-cp310-cp310-linux_x86_64.whl (7.3 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.3/7.3 MB\u001b[0m \u001b[31m41.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting torchaudio\r\n", " Downloading https://download.pytorch.org/whl/cu121/torchaudio-2.5.1%2Bcu121-cp310-cp310-linux_x86_64.whl (3.4 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m16.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting xformers\r\n", " Downloading https://download.pytorch.org/whl/cu121/xformers-0.0.29.post1-cp310-cp310-manylinux_2_28_x86_64.whl (15.3 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m85.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.16.1)\r\n", "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.2)\r\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\r\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\r\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2024.6.1)\r\n", "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m53.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m43.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m90.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-cudnn-cu12==9.1.0.70 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl (664.8 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m664.8/664.8 MB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m13.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m31.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-nccl-cu12==2.21.5 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl (188.7 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m188.7/188.7 MB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting triton==3.1.0 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (209.5 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.5/209.5 MB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting sympy==1.13.1 (from torch)\r\n", " Downloading https://download.pytorch.org/whl/sympy-1.13.1-py3-none-any.whl (6.2 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.2/6.2 MB\u001b[0m \u001b[31m94.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\r\n", " Downloading https://download.pytorch.org/whl/cu121/nvidia_nvjitlink_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (19.8 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.8/19.8 MB\u001b[0m \u001b[31m76.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting mpmath<1.4,>=1.1.0 (from sympy==1.13.1->torch)\r\n", " Downloading https://download.pytorch.org/whl/mpmath-1.3.0-py3-none-any.whl (536 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m536.2/536.2 kB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from torchvision) (1.26.4)\r\n", "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.10/dist-packages (from torchvision) (10.4.0)\r\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\r\n", "Installing collected packages: mpmath, triton, sympy, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, torch, xformers, torchvision, torchaudio\r\n", " Attempting uninstall: nvidia-nccl-cu12\r\n", " Found existing installation: nvidia-nccl-cu12 2.23.4\r\n", " Uninstalling nvidia-nccl-cu12-2.23.4:\r\n", " Successfully uninstalled nvidia-nccl-cu12-2.23.4\r\n", "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n", "fastai 2.7.17 requires torch<2.5,>=1.10, but you have torch 2.5.1+cu121 which is incompatible.\u001b[0m\u001b[31m\r\n", "\u001b[0mSuccessfully installed mpmath-1.3.0 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.21.5 nvidia-nvjitlink-cu12-12.1.105 nvidia-nvtx-cu12-12.1.105 sympy-1.13.1 torch-2.5.1+cu121 torchaudio-2.5.1+cu121 torchvision-0.20.1+cu121 triton-3.1.0 xformers-0.0.29.post1\r\n", "Collecting unsloth\r\n", " Downloading unsloth-2024.12.12-py3-none-any.whl.metadata (60 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.0/60.0 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hCollecting unsloth_zoo>=2024.12.7 (from unsloth)\r\n", " Downloading unsloth_zoo-2024.12.7-py3-none-any.whl.metadata (16 kB)\r\n", "Requirement already satisfied: torch>=2.4.0 in /usr/local/lib/python3.10/dist-packages (from unsloth) (2.5.1+cu121)\r\n", "Requirement already satisfied: xformers>=0.0.27.post2 in /usr/local/lib/python3.10/dist-packages (from unsloth) (0.0.29.post1)\r\n", "Collecting bitsandbytes (from unsloth)\r\n", " Downloading bitsandbytes-0.45.0-py3-none-manylinux_2_24_x86_64.whl.metadata (2.9 kB)\r\n", "Requirement already satisfied: triton>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from unsloth) (3.1.0)\r\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from unsloth) (24.1)\r\n", "Collecting tyro (from unsloth)\r\n", " Downloading tyro-0.9.5-py3-none-any.whl.metadata (9.4 kB)\r\n", "Collecting transformers!=4.47.0,>=4.46.1 (from unsloth)\r\n", " Downloading transformers-4.47.1-py3-none-any.whl.metadata (44 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.1/44.1 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hRequirement already satisfied: datasets>=2.16.0 in /usr/local/lib/python3.10/dist-packages (from unsloth) (3.2.0)\r\n", "Requirement already satisfied: sentencepiece>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from unsloth) (0.2.0)\r\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from unsloth) (4.66.5)\r\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from unsloth) (5.9.5)\r\n", "Requirement already satisfied: wheel>=0.42.0 in /usr/local/lib/python3.10/dist-packages (from unsloth) (0.44.0)\r\n", "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unsloth) (1.26.4)\r\n", "Requirement already satisfied: accelerate>=0.34.1 in /usr/local/lib/python3.10/dist-packages (from unsloth) (0.34.2)\r\n", "Collecting trl!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,>=0.7.9 (from unsloth)\r\n", " Downloading trl-0.13.0-py3-none-any.whl.metadata (11 kB)\r\n", "Collecting peft!=0.11.0,>=0.7.1 (from unsloth)\r\n", " Downloading peft-0.14.0-py3-none-any.whl.metadata (13 kB)\r\n", "Requirement already satisfied: protobuf<4.0.0 in /usr/local/lib/python3.10/dist-packages (from unsloth) (3.20.3)\r\n", "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (from unsloth) (0.24.7)\r\n", "Collecting hf_transfer (from unsloth)\r\n", " Downloading hf_transfer-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (1.7 kB)\r\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.34.1->unsloth) (6.0.2)\r\n", "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.34.1->unsloth) (0.4.5)\r\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (3.16.1)\r\n", "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (18.1.0)\r\n", "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (0.3.8)\r\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (2.1.4)\r\n", "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (2.32.3)\r\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (3.5.0)\r\n", "Requirement already satisfied: multiprocess<0.70.17 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (0.70.16)\r\n", "Requirement already satisfied: fsspec<=2024.9.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets>=2.16.0->unsloth) (2024.6.1)\r\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.16.0->unsloth) (3.10.5)\r\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub->unsloth) (4.12.2)\r\n", "Collecting huggingface_hub (from unsloth)\r\n", " Downloading huggingface_hub-0.27.0-py3-none-any.whl.metadata (13 kB)\r\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (3.3)\r\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (3.1.4)\r\n", "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (12.1.105)\r\n", "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (12.1.105)\r\n", "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (12.1.105)\r\n", "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (9.1.0.70)\r\n", "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (12.1.3.1)\r\n", "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (11.0.2.54)\r\n", "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (10.3.2.106)\r\n", "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (11.4.5.107)\r\n", "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (12.1.0.106)\r\n", "Requirement already satisfied: nvidia-nccl-cu12==2.21.5 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (2.21.5)\r\n", "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (12.1.105)\r\n", "Requirement already satisfied: sympy==1.13.1 in /usr/local/lib/python3.10/dist-packages (from torch>=2.4.0->unsloth) (1.13.1)\r\n", "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=2.4.0->unsloth) (12.1.105)\r\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy==1.13.1->torch>=2.4.0->unsloth) (1.3.0)\r\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers!=4.47.0,>=4.46.1->unsloth) (2024.9.11)\r\n", "Collecting tokenizers<0.22,>=0.21 (from transformers!=4.47.0,>=4.46.1->unsloth)\r\n", " Downloading tokenizers-0.21.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.7 kB)\r\n", "Requirement already satisfied: rich in /usr/local/lib/python3.10/dist-packages (from trl!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,>=0.7.9->unsloth) (13.8.1)\r\n", "Collecting cut_cross_entropy (from unsloth_zoo>=2024.12.7->unsloth)\r\n", " Downloading cut_cross_entropy-24.12.3-py3-none-any.whl.metadata (9.3 kB)\r\n", "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from unsloth_zoo>=2024.12.7->unsloth) (10.4.0)\r\n", "Requirement already satisfied: docstring-parser>=0.15 in /usr/local/lib/python3.10/dist-packages (from tyro->unsloth) (0.16)\r\n", "Collecting shtab>=1.5.6 (from tyro->unsloth)\r\n", " Downloading shtab-1.7.1-py3-none-any.whl.metadata (7.3 kB)\r\n", "Requirement already satisfied: typeguard>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from tyro->unsloth) (4.3.0)\r\n", "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth) (2.4.0)\r\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth) (1.3.1)\r\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth) (24.2.0)\r\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth) (1.4.1)\r\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth) (6.1.0)\r\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth) (1.11.1)\r\n", "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.16.0->unsloth) (4.0.3)\r\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth) (3.3.2)\r\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth) (3.10)\r\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth) (2.2.3)\r\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.32.2->datasets>=2.16.0->unsloth) (2024.8.30)\r\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich->trl!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,>=0.7.9->unsloth) (3.0.0)\r\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich->trl!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,>=0.7.9->unsloth) (2.18.0)\r\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=2.4.0->unsloth) (2.1.5)\r\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->unsloth) (2.8.2)\r\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->unsloth) (2024.2)\r\n", "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets>=2.16.0->unsloth) (2024.1)\r\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich->trl!=0.9.0,!=0.9.1,!=0.9.2,!=0.9.3,>=0.7.9->unsloth) (0.1.2)\r\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets>=2.16.0->unsloth) (1.16.0)\r\n", "Downloading unsloth-2024.12.12-py3-none-any.whl (175 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m175.7/175.7 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading peft-0.14.0-py3-none-any.whl (374 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m374.8/374.8 kB\u001b[0m \u001b[31m16.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading huggingface_hub-0.27.0-py3-none-any.whl (450 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m450.5/450.5 kB\u001b[0m \u001b[31m31.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading transformers-4.47.1-py3-none-any.whl (10.1 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m95.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading trl-0.13.0-py3-none-any.whl (293 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m293.4/293.4 kB\u001b[0m \u001b[31m21.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading unsloth_zoo-2024.12.7-py3-none-any.whl (71 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.4/71.4 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading bitsandbytes-0.45.0-py3-none-manylinux_2_24_x86_64.whl (69.1 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.1/69.1 MB\u001b[0m \u001b[31m26.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading hf_transfer-0.1.8-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m88.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading tyro-0.9.5-py3-none-any.whl (112 kB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m112.5/112.5 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading shtab-1.7.1-py3-none-any.whl (14 kB)\r\n", "Downloading tokenizers-0.21.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.0 MB)\r\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.0/3.0 MB\u001b[0m \u001b[31m98.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n", "\u001b[?25hDownloading cut_cross_entropy-24.12.3-py3-none-any.whl (22 kB)\r\n", "Installing collected packages: shtab, hf_transfer, huggingface_hub, tyro, tokenizers, transformers, cut_cross_entropy, bitsandbytes, peft, trl, unsloth_zoo, unsloth\r\n", " Attempting uninstall: huggingface_hub\r\n", " Found existing installation: huggingface-hub 0.24.7\r\n", " Uninstalling huggingface-hub-0.24.7:\r\n", " Successfully uninstalled huggingface-hub-0.24.7\r\n", " Attempting uninstall: tokenizers\r\n", " Found existing installation: tokenizers 0.19.1\r\n", " Uninstalling tokenizers-0.19.1:\r\n", " Successfully uninstalled tokenizers-0.19.1\r\n", " Attempting uninstall: transformers\r\n", " Found existing installation: transformers 4.44.2\r\n", " Uninstalling transformers-4.44.2:\r\n", " Successfully uninstalled transformers-4.44.2\r\n", "Successfully installed bitsandbytes-0.45.0 cut_cross_entropy-24.12.3 hf_transfer-0.1.8 huggingface_hub-0.27.0 peft-0.14.0 shtab-1.7.1 tokenizers-0.21.0 transformers-4.47.1 trl-0.13.0 tyro-0.9.5 unsloth-2024.12.12 unsloth_zoo-2024.12.7\r\n" ] } ], "source": [ "!pip install pip3-autoremove\n", "!pip-autoremove torch torchvision torchaudio -y\n", "!pip install torch torchvision torchaudio xformers --index-url https://download.pytorch.org/whl/cu121\n", "!pip install unsloth" ] }, { "cell_type": "markdown", "id": "82af8803", "metadata": { "id": "r2v_X2fA0Df5", "papermill": { "duration": 0.055944, "end_time": "2025-01-01T01:39:57.151102", "exception": false, "start_time": "2025-01-01T01:39:57.095158", "status": "completed" }, "tags": [] }, "source": [ "* We support Llama, Mistral, CodeLlama, TinyLlama, Vicuna, Open Hermes etc\n", "* And Yi, Qwen ([llamafied](https://huggingface.co/models?sort=trending&search=qwen+llama)), Deepseek, all Llama, Mistral derived archs.\n", "* We support 16bit LoRA or 4bit QLoRA. Both 2x faster.\n", "* `max_seq_length` can be set to anything, since we do automatic RoPE Scaling via [kaiokendev's](https://kaiokendev.github.io/til) method.\n", "* [**NEW**] With [PR 26037](https://github.com/huggingface/transformers/pull/26037), we support downloading 4bit models **4x faster**! [Our repo](https://huggingface.co/unsloth) has Llama, Mistral 4bit models." ] }, { "cell_type": "code", "execution_count": 2, "id": "81081d4b", "metadata": { "execution": { "iopub.execute_input": "2025-01-01T01:39:57.263299Z", "iopub.status.busy": "2025-01-01T01:39:57.263043Z", "iopub.status.idle": "2025-01-01T01:40:53.320412Z", "shell.execute_reply": "2025-01-01T01:40:53.319479Z" }, "id": "QmUBVEnvCDJv", "outputId": "5eff0d61-05b4-471c-eea2-c2e84a915109", "papermill": { "duration": 56.11539, "end_time": "2025-01-01T01:40:53.322099", "exception": false, "start_time": "2025-01-01T01:39:57.206709", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", "🦥 Unsloth Zoo will now patch everything to make training faster!\n", "==((====))== Unsloth 2024.12.12: Fast Mllama vision patching. Transformers: 4.47.1.\n", " \\\\ /| GPU: Tesla T4. Max memory: 14.741 GB. Platform: Linux.\n", "O^O/ \\_/ \\ Torch: 2.5.1+cu121. CUDA: 7.5. CUDA Toolkit: 12.1. Triton: 3.1.0\n", "\\ / Bfloat16 = FALSE. FA [Xformers = 0.0.29.post1. FA2 = False]\n", " \"-____-\" Free Apache license: http://github.com/unslothai/unsloth\n", "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "e50032cd57a9417892ea70e3fd0a9f5b", "version_major": 2, "version_minor": 0 }, "text/plain": [ "model.safetensors.index.json: 0%| | 0.00/375k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7cf93f9da5bf42e29f601ee3576897c2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Downloading shards: 0%| | 0/2 [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "a5c854bd4bdf4338aa1b628307ae2017", "version_major": 2, "version_minor": 0 }, "text/plain": [ "model-00001-of-00002.safetensors: 0%| | 0.00/4.97G [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "2a377519792d40f1acd33f420c9d0c9d", "version_major": 2, "version_minor": 0 }, "text/plain": [ "model-00002-of-00002.safetensors: 0%| | 0.00/2.94G [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "9b3a216a8e994ec0aa9432cbe1287d2c", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Loading checkpoint shards: 0%| | 0/2 [00:00, ?it/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5589c504ab274effb068bc1f6486acc2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "generation_config.json: 0%| | 0.00/210 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "f73b28abed114385b4e8eaf45bf8dfd2", "version_major": 2, "version_minor": 0 }, "text/plain": [ "preprocessor_config.json: 0%| | 0.00/477 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "41928c4f41b74ffc99afeb8247b05f85", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer_config.json: 0%| | 0.00/55.9k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "594e6d48ef8b46ccafaa6472835c0489", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer.json: 0%| | 0.00/17.2M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7cdaf8e89d0140819d3036c7cffa9a2c", "version_major": 2, "version_minor": 0 }, "text/plain": [ "special_tokens_map.json: 0%| | 0.00/454 [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "4e82f8b403c248618f6d9c141d01f46c", "version_major": 2, "version_minor": 0 }, "text/plain": [ "chat_template.json: 0%| | 0.00/5.15k [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "from unsloth import FastVisionModel\n", "import torch\n", "max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!\n", "dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+\n", "load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.\n", "\n", "# 4bit pre quantized models we support for 4x faster downloading + no OOMs.\n", "fourbit_models = [\n", " \"unsloth/mistral-7b-bnb-4bit\",\n", " \"unsloth/mistral-7b-instruct-v0.2-bnb-4bit\",\n", " \"unsloth/llama-2-7b-bnb-4bit\",\n", " \"unsloth/llama-2-13b-bnb-4bit\",\n", " \"unsloth/codellama-34b-bnb-4bit\",\n", " \"unsloth/tinyllama-bnb-4bit\",\n", " \"unsloth/llama-3-8b-bnb-4bit\",\n", " \"unsloth/llama-3-70b-bnb-4bit\",\n", "] # More models at https://huggingface.co/unsloth\n", "\n", "model, tokenizer = FastVisionModel.from_pretrained(\n", " model_name = \"unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit\", # Choose ANY! eg teknium/OpenHermes-2.5-Mistral-7B\n", " use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for long context\n", " load_in_4bit = load_in_4bit,\n", " # token = \"hf_...\", # use one if using gated models like meta-llama/Llama-2-7b-hf\n", ")" ] }, { "cell_type": "markdown", "id": "f0a8b85e", "metadata": { "id": "SXd9bTZd1aaL", "papermill": { "duration": 0.062277, "end_time": "2025-01-01T01:40:53.443926", "exception": false, "start_time": "2025-01-01T01:40:53.381649", "status": "completed" }, "tags": [] }, "source": [ "We now add LoRA adapters so we only need to update 1 to 10% of all parameters!" ] }, { "cell_type": "code", "execution_count": 3, "id": "8f93fa2c", "metadata": { "execution": { "iopub.execute_input": "2025-01-01T01:40:53.602745Z", "iopub.status.busy": "2025-01-01T01:40:53.602423Z", "iopub.status.idle": "2025-01-01T01:40:59.210658Z", "shell.execute_reply": "2025-01-01T01:40:59.209971Z" }, "id": "6bZsfBuZDeCL", "outputId": "b630cc80-ff95-45a2-cc0d-38666010d73b", "papermill": { "duration": 5.707488, "end_time": "2025-01-01T01:40:59.212195", "exception": false, "start_time": "2025-01-01T01:40:53.504707", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "model = FastVisionModel.get_peft_model(\n", " model,\n", " r = 32, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128\n", " finetune_vision_layers = True, # False if not finetuning vision part\n", " finetune_language_layers = True, # False if not finetuning language part\n", " finetune_attention_modules = True, # False if not finetuning attention layers\n", " finetune_mlp_modules = True, # False if not finetuning MLP layers\n", " lora_alpha = 32, # Recommended alpha == r at least\n", " lora_dropout = 0, # Supports any, but = 0 is optimized\n", " bias = \"none\", # Supports any, but = \"none\" is optimized\n", " random_state = 3407,\n", " use_rslora = False, # We support rank stabilized LoRA\n", " loftq_config = None, # And LoftQ\n", ")" ] }, { "cell_type": "markdown", "id": "bbe9e44b", "metadata": { "id": "vITh0KVJ10qX", "papermill": { "duration": 0.05753, "end_time": "2025-01-01T01:40:59.328848", "exception": false, "start_time": "2025-01-01T01:40:59.271318", "status": "completed" }, "tags": [] }, "source": [ "\n", "### Data Prep\n", "We now use the Alpaca dataset from [yahma](https://huggingface.co/datasets/yahma/alpaca-cleaned), which is a filtered version of 52K of the original [Alpaca dataset](https://crfm.stanford.edu/2023/03/13/alpaca.html). You can replace this code section with your own data prep.\n", "\n", "**[NOTE]** To train only on completions (ignoring the user's input) read TRL's docs [here](https://huggingface.co/docs/trl/sft_trainer#train-on-completions-only).\n", "\n", "**[NOTE]** Remember to add the **EOS_TOKEN** to the tokenized output!! Otherwise you'll get infinite generations!\n", "\n", "If you want to use the `ChatML` template for ShareGPT datasets, try our conversational [notebook](https://colab.research.google.com/drive/1Aau3lgPzeZKQ-98h69CCu1UJcvIBLmy2?usp=sharing).\n", "\n", "For text completions like novel writing, try this [notebook](https://colab.research.google.com/drive/1ef-tab5bhkvWmBOObepl1WgJvfvSzn5Q?usp=sharing)." ] }, { "cell_type": "code", "execution_count": 4, "id": "3a74fde3", "metadata": { "execution": { "iopub.execute_input": "2025-01-01T01:40:59.444214Z", "iopub.status.busy": "2025-01-01T01:40:59.443934Z", "iopub.status.idle": "2025-01-01T01:41:04.410001Z", "shell.execute_reply": "2025-01-01T01:41:04.409044Z" }, "id": "LjY75GoYUCB8", "outputId": "9f40f734-788c-4793-c1af-e9d003337612", "papermill": { "duration": 5.025641, "end_time": "2025-01-01T01:41:04.411709", "exception": false, "start_time": "2025-01-01T01:40:59.386068", "status": "completed" }, "tags": [] }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "0aca87e0f7d4459ea312cf1a7a3160a4", "version_major": 2, "version_minor": 0 }, "text/plain": [ "image_summaries.parquet: 0%| | 0.00/264M [00:00, ?B/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "e3044302cb2e43aeae8b2aa8ee9d0006", "version_major": 2, "version_minor": 0 }, "text/plain": [ "Generating train split: 0%| | 0/183 [00:00, ? examples/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "/usr/local/lib/python3.10/dist-packages/PIL/Image.py:3368: DecompressionBombWarning: Image size (101183704 pixels) exceeds limit of 89478485 pixels, could be decompression bomb DOS attack.\n", " warnings.warn(\n", "/usr/local/lib/python3.10/dist-packages/PIL/Image.py:3368: DecompressionBombWarning: Image size (101192432 pixels) exceeds limit of 89478485 pixels, could be decompression bomb DOS attack.\n", " warnings.warn(\n" ] } ], "source": [ "from datasets import load_dataset, concatenate_datasets\n", "from unsloth.chat_templates import get_chat_template\n", "from PIL import Image\n", "import io\n", "import json\n", "import random\n", "\n", "booru = load_dataset(\"scoliono/fembooru\", split=\"train\")\n", "\n", "visual_instructions = [\"Describe this image.\", \"What's going on in this picture?\", \"What is this?\", \"What's happening in this image?\", \"Tell me what's in this picture.\", \"Describe what's happening here.\"]\n", "\n", "def convert_to_conversation(sample):\n", " conversation = [\n", " { \"role\": \"user\",\n", " \"content\" : [\n", " {\"type\" : \"text\", \"text\" : random.choice(visual_instructions)},\n", " {\"type\" : \"image\", \"image\" : Image.open(io.BytesIO(sample[\"image_data\"]))} ]\n", " },\n", " { \"role\" : \"assistant\",\n", " \"content\" : [\n", " {\"type\" : \"text\", \"text\" : sample[\"summary\"]} ]\n", " },\n", " ]\n", " return { \"messages\" : conversation }\n", "\n", "def formatting_prompts_func(convos):\n", " texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for convo in convos]\n", " return { \"text\" : texts, }\n", "\n", "with open(\"/kaggle/input/the-group-chat/output-10k-c-dropout-nonames-replies.json\") as chatfile:\n", " convos = [json.loads(j) for j in chatfile.readlines()]\n", "\n", "with open(\"/kaggle/input/toxicqa/toxicQAfinal.json\") as chatfile:\n", " convos += [json.loads(j) for j in chatfile.readlines()]\n", "\n", "# shim to convert our text-only json format into one consistent with multimodal training data\n", "def text_to_visual_convo(convo):\n", " return [\n", " { \"role\": msg[\"role\"], \"content\": [{\"type\": \"text\", \"text\": msg[\"content\"]}] }\n", " for msg in convo\n", " ]\n", "\n", "fmt_convos = [ {\"messages\": text_to_visual_convo(convo)} for convo in convos ]\n", "fmt_booru = [ convert_to_conversation(sample) for sample in booru ]\n", "dataset = fmt_booru #+ fmt_convos" ] }, { "cell_type": "markdown", "id": "b12b1e45", "metadata": { "id": "idAEIeSQ3xdS", "papermill": { "duration": 0.058306, "end_time": "2025-01-01T01:41:04.529231", "exception": false, "start_time": "2025-01-01T01:41:04.470925", "status": "completed" }, "tags": [] }, "source": [ "\n", "### Train the model\n", "Now let's use Huggingface TRL's `SFTTrainer`! More docs here: [TRL SFT docs](https://huggingface.co/docs/trl/sft_trainer). We do 60 steps to speed things up, but you can set `num_train_epochs=1` for a full run, and turn off `max_steps=None`. We also support TRL's `DPOTrainer`!" ] }, { "cell_type": "code", "execution_count": 5, "id": "116e1ce1", "metadata": { "execution": { "iopub.execute_input": "2025-01-01T01:41:04.644664Z", "iopub.status.busy": "2025-01-01T01:41:04.644382Z", "iopub.status.idle": "2025-01-01T01:41:06.493679Z", "shell.execute_reply": "2025-01-01T01:41:06.493023Z" }, "id": "95_Nn-89DhsL", "outputId": "4b809e6d-271f-446f-dec8-abe0d13259f8", "papermill": { "duration": 1.908619, "end_time": "2025-01-01T01:41:06.495152", "exception": false, "start_time": "2025-01-01T01:41:04.586533", "status": "completed" }, "tags": [] }, "outputs": [], "source": [ "from trl import SFTTrainer, SFTConfig\n", "from unsloth.trainer import UnslothVisionDataCollator\n", "from transformers import DataCollatorForSeq2Seq\n", "from unsloth import is_bf16_supported\n", "from transformers import AutoProcessor\n", "\n", "FastVisionModel.for_training(model) # Enable for training!\n", "\n", "processor = AutoProcessor.from_pretrained(\"unsloth/Llama-3.2-11B-Vision-Instruct-unsloth-bnb-4bit\")\n", "\n", "# unsloth-zoo collator doesn't support messages without images\n", "def collate_fn(examples):\n", " # Get the texts and images, and apply the chat template\n", " texts = [processor.apply_chat_template(example[\"messages\"], tokenize=False) for example in examples]\n", " # MUST be None or be populated with images; this is what unsloth is missing\n", " images = None\n", " for example in examples:\n", " if \"messages\" in examples:\n", " for msg in examples[\"messages\"]:\n", " if isinstance(msg[\"content\"], list):\n", " has_image = False\n", " for mode in msg[\"content\"]:\n", " if mode[\"type\"] == \"image\":\n", " if images is None:\n", " images = []\n", " images.append(mode[\"image\"])\n", " has_image = True\n", " if not has_image and isinstance(images, list):\n", " images.append(None)\n", "\n", " # Tokenize the texts and process the images\n", " batch = processor(texts, images, return_tensors=\"pt\", padding=True)\n", "\n", " # The labels are the input_ids, and we mask the padding tokens in the loss computation\n", " labels = batch[\"input_ids\"].clone()\n", " labels[labels == processor.tokenizer.pad_token_id] = -100\n", " batch[\"labels\"] = labels\n", "\n", " return batch\n", "\n", "trainer = SFTTrainer(\n", " model = model,\n", " tokenizer = tokenizer,\n", " train_dataset = dataset,\n", " #data_collator = collate_fn,\n", " data_collator = UnslothVisionDataCollator(model, tokenizer),\n", " dataset_text_field = \"text\",\n", " max_seq_length = max_seq_length,\n", " dataset_num_proc = 2,\n", " packing = False, # Can make training 5x faster for short sequences.\n", " args = SFTConfig(\n", " per_device_train_batch_size = 2,\n", " gradient_accumulation_steps = 4,\n", " warmup_steps = 5,\n", " #max_steps = 30,\n", " num_train_epochs = 1, # Set this instead of max_steps for full training runs\n", " learning_rate = 2e-4,\n", " fp16 = not is_bf16_supported(),\n", " bf16 = is_bf16_supported(),\n", " logging_steps = 1,\n", " optim = \"adamw_8bit\",\n", " weight_decay = 0.01,\n", " lr_scheduler_type = \"linear\",\n", " seed = 3407,\n", " output_dir = \"outputs\",\n", " report_to = \"none\", # For Weights and Biases\n", "\n", " # You MUST put the below items for vision finetuning:\n", " remove_unused_columns = False,\n", " dataset_text_field = \"\",\n", " dataset_kwargs = {\"skip_prepare_dataset\": True},\n", " dataset_num_proc = 4,\n", " max_seq_length = 2048,\n", " ),\n", ")" ] }, { "cell_type": "code", "execution_count": 6, "id": "962391b6", "metadata": { "cellView": "form", "execution": { "iopub.execute_input": "2025-01-01T01:41:06.614544Z", "iopub.status.busy": "2025-01-01T01:41:06.614275Z", "iopub.status.idle": "2025-01-01T01:41:06.619459Z", "shell.execute_reply": "2025-01-01T01:41:06.618801Z" }, "id": "2ejIt2xSNKKp", "outputId": "4815a050-0c0f-4a6a-9d93-b01c44eaea35", "papermill": { "duration": 0.06503, "end_time": "2025-01-01T01:41:06.620624", "exception": false, "start_time": "2025-01-01T01:41:06.555594", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "GPU = Tesla T4. Max memory = 14.741 GB.\n", "7.881 GB of memory reserved.\n" ] } ], "source": [ "#@title Show current memory stats\n", "gpu_stats = torch.cuda.get_device_properties(0)\n", "start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)\n", "max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)\n", "print(f\"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.\")\n", "print(f\"{start_gpu_memory} GB of memory reserved.\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "52bb19f4", "metadata": { "execution": { "iopub.execute_input": "2025-01-01T01:41:06.736648Z", "iopub.status.busy": "2025-01-01T01:41:06.736441Z", "iopub.status.idle": "2025-01-01T01:55:19.558624Z", "shell.execute_reply": "2025-01-01T01:55:19.557921Z" }, "id": "yqxqAZ7KJ4oL", "outputId": "3cf26aac-6042-4458-c4a6-d8849efb6a95", "papermill": { "duration": 852.881724, "end_time": "2025-01-01T01:55:19.560033", "exception": false, "start_time": "2025-01-01T01:41:06.678309", "status": "completed" }, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "==((====))== Unsloth - 2x faster free finetuning | Num GPUs = 1\n", " \\\\ /| Num examples = 183 | Num Epochs = 1\n", "O^O/ \\_/ \\ Batch size per device = 2 | Gradient Accumulation steps = 4\n", "\\ / Total batch size = 8 | Total steps = 23\n", " \"-____-\" Number of trainable parameters = 134,348,800\n", "🦥 Unsloth needs about 1-3 minutes to load everything - please wait!\n" ] }, { "data": { "text/html": [ "\n", "Step | \n", "Training Loss | \n", "
---|---|
1 | \n", "2.972900 | \n", "
2 | \n", "2.893800 | \n", "
3 | \n", "2.855700 | \n", "
4 | \n", "2.607000 | \n", "
5 | \n", "2.517700 | \n", "
6 | \n", "1.886900 | \n", "
7 | \n", "1.887000 | \n", "
8 | \n", "1.712500 | \n", "
9 | \n", "1.482400 | \n", "
10 | \n", "1.447500 | \n", "
11 | \n", "1.375100 | \n", "
12 | \n", "1.479200 | \n", "
13 | \n", "1.241500 | \n", "
14 | \n", "1.298700 | \n", "
15 | \n", "1.174100 | \n", "
16 | \n", "1.193000 | \n", "
17 | \n", "1.151100 | \n", "
18 | \n", "1.173700 | \n", "
19 | \n", "0.891000 | \n", "
20 | \n", "0.976000 | \n", "
21 | \n", "1.198000 | \n", "
22 | \n", "1.212600 | \n", "
23 | \n", "1.119600 | \n", "
"
],
"text/plain": [
"