conda install cudatoolkit xformers bitsandbytes pytorch pytorch-cuda=12.1 \

  -c pytorch -c nvidia -c xformers -c conda-forge -y

pip install "unsloth[conda] @ git+https://github.com/unslothai/unsloth.git"

import torch; torch.version.cuda

pip install --upgrade --force-reinstall --no-cache-dir torch==2.1.0 triton \

  --index-url https://download.pytorch.org/whl/cu121

pip install "unsloth[cu118] @ git+https://github.com/unslothai/unsloth.git"

pip install "unsloth[cu121] @ git+https://github.com/unslothai/unsloth.git"

pip install "unsloth[cu118_ampere] @ git+https://github.com/unslothai/unsloth.git"

pip install "unsloth[cu121_ampere] @ git+https://github.com/unslothai/unsloth.git"

pip install --upgrade --force-reinstall --no-cache-dir torch==2.1.1 triton \

  --index-url https://download.pytorch.org/whl/cu121

pip install "unsloth[cu118_torch211] @ git+https://github.com/unslothai/unsloth.git"

pip install "unsloth[cu121_torch211] @ git+https://github.com/unslothai/unsloth.git"

pip install "unsloth[cu118_ampere_torch211] @ git+https://github.com/unslothai/unsloth.git"

pip install "unsloth[cu121_ampere_torch211] @ git+https://github.com/unslothai/unsloth.git"

pip install --upgrade pip

from unsloth import FastLanguageModel

import torch

from trl import SFTTrainer

from transformers import TrainingArguments

from datasets import load_dataset

max_seq_length = 2048 # Supports RoPE Scaling interally, so choose any!

# Get LAION dataset

url = "https://huggingface.co/datasets/laion/OIG/resolve/main/unified_chip2.jsonl"

dataset = load_dataset("json", data_files = {"train" : url}, split = "train")

# 4bit pre quantized models we support - 4x faster downloading!

fourbit_models = [

    "unsloth/mistral-7b-bnb-4bit",

    "unsloth/llama-2-7b-bnb-4bit",

    "unsloth/llama-2-13b-bnb-4bit",

    "unsloth/codellama-34b-bnb-4bit",

    "unsloth/tinyllama-bnb-4bit",

]

# Load Llama model

model, tokenizer = FastLanguageModel.from_pretrained(

    model_name = "unsloth/mistral-7b-bnb-4bit", # Supports Llama, Mistral - replace this!

    max_seq_length = max_seq_length,

    dtype = None,

    load_in_4bit = True,

)

# Do model patching and add fast LoRA weights

model = FastLanguageModel.get_peft_model(

    model,

    r = 16,

    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",

                      "gate_proj", "up_proj", "down_proj",],

    lora_alpha = 16,

    lora_dropout = 0, # Currently only supports dropout = 0

    bias = "none",    # Currently only supports bias = "none"

    use_gradient_checkpointing = True,

    random_state = 3407,

    max_seq_length = max_seq_length,

)

trainer = SFTTrainer(

    model = model,

    train_dataset = dataset,

    dataset_text_field = "text",

    max_seq_length = max_seq_length,

    tokenizer = tokenizer,

    args = TrainingArguments(

        per_device_train_batch_size = 2,

        gradient_accumulation_steps = 4,

        warmup_steps = 10,

        max_steps = 60,

        fp16 = not torch.cuda.is_bf16_supported(),

        bf16 = torch.cuda.is_bf16_supported(),

        logging_steps = 1,

        output_dir = "outputs",

        optim = "adamw_8bit",

        seed = 3407,

    ),

)

trainer.train()

from unsloth import FastLanguageModel, PatchDPOTrainer

PatchDPOTrainer()

import torch

from transformers import TrainingArguments

from trl import DPOTrainer

model, tokenizer = FastLanguageModel.from_pretrained(

    model_name = "unsloth/zephyr-sft-bnb-4bit",

    max_seq_length = max_seq_length,

    dtype = None,

    load_in_4bit = True,

)

# Do model patching and add fast LoRA weights

model = FastLanguageModel.get_peft_model(

    model,

    r = 64,

    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",

                      "gate_proj", "up_proj", "down_proj",],

    lora_alpha = 64,

    lora_dropout = 0, # Currently only supports dropout = 0

    bias = "none",    # Currently only supports bias = "none"

    use_gradient_checkpointing = True,

    random_state = 3407,

    max_seq_length = max_seq_length,

)

dpo_trainer = DPOTrainer(

    model = model,

    ref_model = None,

    args = TrainingArguments(

        per_device_train_batch_size = 4,

        gradient_accumulation_steps = 8,

        warmup_ratio = 0.1,

        num_train_epochs = 3,

        fp16 = not torch.cuda.is_bf16_supported(),

        bf16 = torch.cuda.is_bf16_supported(),

        logging_steps = 1,

        optim = "adamw_8bit",

        seed = 42,

        output_dir = "outputs",

    ),

    beta = 0.1,

    train_dataset = YOUR_DATASET_HERE,

    # eval_dataset = YOUR_DATASET_HERE,

    tokenizer = tokenizer,

    max_length = 1024,

    max_prompt_length = 512,

)

dpo_trainer.train()