微调Qwen2.5-coder，只用GRPO！打造能“懂你时间表”的AI工具人！

from unsloth import FastLanguageModelmax_seq_length = 2048lora_rank = 32
model, tokenizer = FastLanguageModel.from_pretrained(    model_name = "Qwen/Qwen2.5-Coder-7B-Instruct",    max_seq_length = max_seq_length,    load_in_4bit = True,    fast_inference = True,    max_lora_rank = lora_rank,    gpu_memory_utilization = 0.85, # Reduce if out of memory)
model = FastLanguageModel.get_peft_model(    model,    r = lora_rank,    target_modules = [        "q_proj", "k_proj", "v_proj", "o_proj",        "gate_proj", "up_proj", "down_proj",    ], # Remove QKVO if out of memory    lora_alpha = lora_rank,    use_gradient_checkpointing = "unsloth", # Enable long context finetuning    random_state = 3407,)

import datasets
SYSTEM_PROMPT = """You are a precise event scheduler.1. First, reason through the problem inside <think> and </think> tags. Here you can create drafts, compare alternatives, and check for mistakes.2. When confident, output the final schedule inside <schedule> and </schedule> tags. Your schedule must strictly follow the rules provided by the user."""
USER_PROMPT ="""Task: create an optimized schedule based on the given events.
Rules:- The schedule MUST be in strict chronological order.   Do NOT place priority events earlier unless their actual start time is earlier.- Event start and end times are ABSOLUTE. NEVER change, shorten, adjust, or split them.- Priority events (weight = 2) carry more weight than normal events (weight = 1),   but they MUST still respect chronological order.- Maximize the sum of weighted event durations.- No overlaps allowed. In conflicts, include the event with the higher weighted time.- Some events may be excluded if needed to meet these rules.

You must use this format:  
<think>...</think><schedule><event><name>...</name><start>...</start><end>...</end></event>...</schedule>
---
"""
ds = datasets.load_dataset("anakin87/events-scheduling", split="train")
ds = ds.map(    lambda x: {        "prompt": [            {"role": "system", "content": SYSTEM_PROMPT},            {"role": "user", "content": USER_PROMPT + x["prompt"]},        ]    })

import re
overall_pattern = (r"<think>.+</think>.*<schedule>.*(<event>.*<name>.+</name>.*<start>\d{2}:\d{2}</start>.*"                   r"<end>\d{2}:\d{2}</end>.*</event>)+.*</schedule>")overall_regex = re.compile(overall_pattern, re.DOTALL)
def format_reward(prompts, completions, **kwargs):    responses = [completion[0]['content'] for completion in completions]    return [0.0 if not overall_regex.match(response) else 10.0 for response in responses]

def sorted_events_reward(completions, **kwargs):    scores = []    responses = [completion[0]['content'] for completion in completions]
    for response in responses:
        scheduled_events = get_events(response)
        # not a valid schedule: should be discarded        if len(scheduled_events) < 2:            scores.append(0.0)            continue
        scheduled_events_minutes = [(ev[0], time_to_minutes(ev[1]), time_to_minutes(ev[2]))                                   for ev in scheduled_events]
        if all(scheduled_events_minutes[i][1] < scheduled_events_minutes[i+1][1]                   for i in range(len(scheduled_events_minutes)-1)):            scores.append(20.0)        else:            scores.append(0)
    return scores

def score_reward(prompts, completions, events, priority_events, optimal_score, **kwargs):    scores = []    responses = [completion[0]['content'] for completion in completions]
    for content, valid_events, priorities, opt_score in zip(responses, events, priority_events, optimal_score):
        scheduled_events = get_events(content)
        # Get valid scheduled events        existing_events = {ev for ev in scheduled_events if [ev[0], ev[1], ev[2]] in valid_events}
        # penalize choosing nonexistent events or less than 2 events (not a valid schedule)        if len(existing_events)<len(scheduled_events) or len(existing_events) < 2:            scores.append(0.0)            continue
        # Convert to minutes        existing_events_minutes = [(ev[0], time_to_minutes(ev[1]), time_to_minutes(ev[2]))                                   for ev in existing_events]

        # remove overlapping events and remove both events - to penalize overlaps        overlapping_events = set()        for j in range(len(existing_events_minutes)):            for k in range(j + 1, len(existing_events_minutes)):                if (existing_events_minutes[j][1] <= existing_events_minutes[k][2] and                     existing_events_minutes[j][2] >= existing_events_minutes[k][1]):                    overlapping_events.add(existing_events_minutes[j])                    overlapping_events.add(existing_events_minutes[k])
        existing_events_minutes = [ev for ev in existing_events_minutes                                   if ev not in overlapping_events]
        # Calculate score        score = sum(2 * (ev[2] - ev[1]) if ev[0] in priorities                    else ev[2] - ev[1] for ev in existing_events_minutes)
        scores.append((score/opt_score) * 70)
    return scores

from trl import GRPOConfig, GRPOTrainer
tokenized_prompts = [tokenizer.apply_chat_template(prompt, tokenize=True, add_generation_prompt=True)                       for prompt in ds['prompt']]exact_max_prompt_length = max([len(tokenized_prompt) for tokenized_prompt in tokenized_prompts])
max_prompt_length = 448  # manually adjustednew_model_id="anakin87/qwen-scheduler-7b-grpo"
training_args = GRPOConfig(    learning_rate = 8e-6,    adam_beta1 = 0.9,    adam_beta2 = 0.99,    weight_decay = 0.1,    warmup_ratio = 0.01,    lr_scheduler_type = "cosine",    optim = "paged_adamw_8bit",    logging_steps = 1,    per_device_train_batch_size = 8,    gradient_accumulation_steps = 1,    num_generations = 8, # Decrease if out of memory    max_prompt_length = max_prompt_length,    max_completion_length = max_seq_length - max_prompt_length,    max_grad_norm = 0.1,    output_dir = "outputs",    overwrite_output_dir = True,    push_to_hub = True,    hub_model_id=new_model_id,    hub_strategy="every_save",    save_strategy="steps",    save_steps=50,    save_total_limit=1,    num_train_epochs=3,)
trainer = GRPOTrainer(    model = model,    processing_class = tokenizer,    reward_funcs=[        format_reward,        sorted_events_reward,        score_reward,    ],    args = training_args,    train_dataset = ds,)trainer.train()