Formation GPT-2 pour générer du haïku

import csv
import re

with open('all_haiku.csv', 'r') as csv_file:
    reader = csv.reader(csv_file)
    i = 0
    txt_file = open('input.txt', 'w')
    for row in reader:
        strings = row[1:4]
        upd_strings = []
        for str in strings:
            if len(str) > 0:
                if str[0] == ' ':
                    while str[0] == ' ':
                        str = str[1:]
                if str[-1] == ' ':
                    while str[-1] == ' ':
                        str = str[:-1] 
                upd_strings.append(re.sub(r"[^a-zA-Z ]+", "", str)) 
            else:
                upd_strings.append('')
        # skip text and label 
        if i > 0: 
            txt_file.write(upd_strings[0] + ';' + upd_strings[1] + ';' + upd_strings[2] +'\n')
        i+=1

    print("Added", i-1, "strings of text")

import time

out_dir="out-haiku"
eval_interval = 5
eval_iters = 40
wandb_log = True # feel free to turn on
wandb_project="haiku"
wandb_run_name="test3"

dataset="haiku"
init_from = 'gpt2-large' # this is the largest GPT-2 model

# only save checkpoints if the validation loss improves
always_save_checkpoint = False

# the number of examples per iter:
# 1 batch_size * 32 grad_accum * 1024 tokens = 32,768 tokens/iter
# shakespeare has 301,966 tokens, so 1 epoch ~= 9.2 iters
# haiku has 2,301,966 tokens, so 1 epoch ~= 70 iters * 10 number of iterations
batch_size = 1
gradient_accumulation_steps = 32
max_iters = 1000

# finetune at constant LR
learning_rate = 1e-6
decay_lr = True
warmup_iters = 200#max_iters/10
lr_decay_iters = max_iters 
min_lr = learning_rate/10

compile=False

 » ou etc. Peut également spécifier un fichier, à utiliser comme : « FILE:prompt.txt » num_samples = 10 # nombre d’échantillons à dessiner max_new_tokens = 20 # nombre de jetons générés dans chaque échantillon temperature = 0,9 # 1,0 = aucun changement, 1,0 = plus aléatoire, dans les prédictions top_k = 200 # ne conserver que les top_k jetons les plus probables, bloquer les autres pour avoir 0 graine de probabilité = 1381 device= »cuda » # exemples : ‘cpu’, ‘cuda’, ‘cuda:0’, ‘ cuda:1’, etc. dtype= »bfloat16″ # ‘float32’ ou ‘bfloat16’ ou ‘float16’ compile = False # utilise PyTorch 2.0 pour compiler le modèle plus rapidement exec(open(‘configurator.py’).read ()) # remplace la ligne de commande ou le fichier de configuration # ————————————– ————————————— torch.manual_seed(graine) torch.cuda.manual_seed (graine) torch.backends.cuda.matmul.allow_tf32 = True # autoriser tf32 sur matmul torch.backends.cudnn.allow_tf32 = True # autoriser tf32 sur cudnn device_type= »cuda » if ‘cuda’ in device else ‘cpu’ # for utilisation ultérieure dans torch.autocast ptdtype = {‘float32’: torch.float32, ‘ bfloat16’ : torche.bfloat16, ‘float16’ : torche.float16}[dtype]
ctx = nullcontext() if device_type == ‘cpu’ else torch.amp.autocast(device_type=device_type, dtype=ptdtype) # model if init_from == ‘resume’: # init depuis un modèle enregistré dans un répertoire spécifique ckpt_path = os .path.join(out_dir, ‘ckpt.pt’) checkpoint = torch.load(ckpt_path, map_location=device) gptconf = GPTConfig(**checkpoint[‘model_args’]) modèle = GPT(gptconf) state_dict = point de contrôle[‘model’]
préfixe_indésirable = ‘_orig_mod.’ for k,v in list(state_dict.items()): if k.startswith(unwanted_prefix): state_dict[k[len(unwanted_prefix):]]= state_dict.pop(k) model.load_state_dict(state_dict) elif init_from.startswith(‘gpt2’): # init d’un modèle GPT-2 donné model = GPT.from_pretrained(init_from, dict(dropout=0.0)) model. eval() model.to(device) if compile: model = torch.compile(model) # requiert PyTorch 2.0 (optionnel) # recherche le meta pickle s’il est disponible dans le dossier du jeu de données load_meta = False if init_from == ‘ resume’ et ‘config’ dans checkpoint et ‘dataset’ dans checkpoint[‘config’]: # les points de contrôle plus anciens peuvent ne pas avoir ces… meta_path = os.path.join(‘data’, checkpoint[‘config’][‘dataset’]’meta.pkl’) load_meta = os.path.exists(meta_path) if load_meta: print(f »Loading meta from {meta_path}… ») with open(meta_path, ‘rb’) as f: meta = pickle .load(f) # TODO veut rendre cela plus général aux schémas d’encodeurs/décodeurs arbitraires stoi, itos = meta[‘stoi’]méta[‘itos’]
encoder = lambda s : [stoi[c] for c in s]decode = lambda l:  ».join([itos[i] for i in l]) else : # ok supposons les encodages gpt-2 par défaut print(« Aucun meta.pkl trouvé, en supposant les encodages GPT-2… ») enc = tiktoken.get_encoding(« gpt2 ») encode = lambda s: enc.encode(s, allow_special={«  »}) decode = lambda l: enc.decode(l) # encode le début de l’invite #if start.startswith(‘FILE:’): # with open(start[5:]’r’, encoding=’utf-8′) as f: # start = f.read() start_ids = encode(promt) x = (torch.tensor(start_ids, dtype=torch.long, device=device)[None, …]) print(‘—————‘) # lancer la génération avec torch.no_grad() : with ctx : for k in range(num_samples) : y = model.generate(x, max_new_tokens, temperature=temperature, top_k=top_k) print(decode(y[0].tolist())) print(‘—————‘) » data-lang= »text/x-python »>

"""
Sample from a trained model
"""
import os
import pickle
from contextlib import nullcontext
import torch
import tiktoken
from model import GPTConfig, GPT

promt = "Full Moon is shining\n"

# -----------------------------------------------------------------------------
init_from = 'resume' # either 'resume' (from an out_dir) or a gpt2 variant (e.g. 'gpt2-xl')
out_dir="out-haiku_1k" # ignored if init_from is not 'resume'
start = "\n" # or "<|endoftext|>" or etc. Can also specify a file, use as: "FILE:prompt.txt"
num_samples = 10 # number of samples to draw
max_new_tokens = 20 # number of tokens generated in each sample
temperature = 0.9 # 1.0 = no change, < 1.0 = less random, > 1.0 = more random, in predictions
top_k = 200 # retain only the top_k most likely tokens, clamp others to have 0 probability
seed = 1381
device="cuda" # examples: 'cpu', 'cuda', 'cuda:0', 'cuda:1', etc.
dtype="bfloat16" # 'float32' or 'bfloat16' or 'float16'
compile = False # use PyTorch 2.0 to compile the model to be faster
exec(open('configurator.py').read()) # overrides from command line or config file
# -----------------------------------------------------------------------------

torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cuda.matmul.allow_tf32 = True # allow tf32 on matmul
torch.backends.cudnn.allow_tf32 = True # allow tf32 on cudnn
device_type="cuda" if 'cuda' in device else 'cpu' # for later use in torch.autocast
ptdtype = {'float32': torch.float32, 'bfloat16': torch.bfloat16, 'float16': torch.float16}[dtype]
ctx = nullcontext() if device_type == 'cpu' else torch.amp.autocast(device_type=device_type, dtype=ptdtype)

# model
if init_from == 'resume':
    # init from a model saved in a specific directory
    ckpt_path = os.path.join(out_dir, 'ckpt.pt')
    checkpoint = torch.load(ckpt_path, map_location=device)
    gptconf = GPTConfig(**checkpoint['model_args'])
    model = GPT(gptconf)
   ...