daglie

app.py

@@ -1,42 +1,41 @@
-import pandas as pd
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer
-import csv
-import yaml
+from transformers import AutoModelForCausalLM, AutoTokenizer, TrainingArguments, Trainer, BertLMHeadModel, BertForSequenceClassification
 from datasets import Dataset
+import pandas as pd
+import csv
+from transformers import TrainingArguments, Trainer
 
 import tensorflow as tf
 # Check TensorFlow GPU availability
 print("GPUs Available: ", tf.config.list_physical_devices('GPU'))
 
 import os
+# Setting the environment variable for MPS
 os.environ['PYTORCH_MPS_HIGH_WATERMARK_RATIO'] = '0.0'
 
+def get_device():
+    """Automatically chooses the best device."""
+    if torch.cuda.is_available():
+        return torch.device('cuda')
+    elif torch.backends.mps.is_available():
+        return torch.device('mps')
+    else:
+        return torch.device('cpu')
+
 def load_data_and_config(data_path):
     """Loads training data from CSV."""
     data = []
     with open(data_path, newline='', encoding='utf-8') as csvfile:
-        reader = csv.DictReader(csvfile, delimiter=';')  # Ensure delimiter matches your CSV file
+        reader = csv.DictReader(csvfile, delimiter=';')
         for row in reader:
-            data.append({'text': row['description']})  # Changed from 'text' to 'description'
+            data.append({'text': row['description']})
     return data
 
-def generate_api_query(model, tokenizer, prompt, desired_output, api_name, base_url):
-    """Generates an API query using a fine-tuned model."""
-    input_ids = tokenizer.encode(prompt + f" Write an API query to {api_name} to get {desired_output}", return_tensors="pt")
-    input_ids = input_ids.to(model.device)  # Ensure input_ids are on the same device as the model
-    output = model.generate(input_ids, max_length=256, temperature=0.7, do_sample=True)  # Enable sampling with temperature control
-    query = tokenizer.decode(output[0], skip_special_tokens=True)
-    return f"{base_url}/{query}"
-
-from transformers import TrainingArguments, Trainer
-
-def train_model(model, tokenizer, data):
+def train_model(model, tokenizer, data, device):
     """Trains the model using the Hugging Face Trainer API."""
-    # Encode data and prepare labels
     inputs = [tokenizer(d['text'], max_length=512, truncation=True, padding='max_length', return_tensors="pt") for d in data]
     dataset = Dataset.from_dict({
-        'input_ids': [x['input_ids'].squeeze() for x in inputs],  # remove extra dimensions
+        'input_ids': [x['input_ids'].squeeze() for x in inputs],
         'labels': [x['input_ids'].squeeze() for x in inputs]
     })
     
@@ -50,47 +49,53 @@ def train_model(model, tokenizer, data):
         logging_dir='./logs',
         logging_steps=10,
     )
-
+    
     trainer = Trainer(
         model=model,
         args=training_args,
         train_dataset=dataset,
         tokenizer=tokenizer
     )
-
-    # The Trainer handles the training loop internally
+    
     trainer.train()
 
-    # Optionally clear cache if using GPU or MPS
-    if torch.cuda.is_available():
-        torch.cuda.empty_cache()
-    elif torch.backends.mps.is_built():
-        torch.mps.empty_cache()
-
     # Perform any remaining steps such as logging, saving, etc.
     trainer.save_model()
 
 def main(api_name, base_url):
-    # Load data
+    device = get_device()  # Get the appropriate device
     data = load_data_and_config("train2.csv")
+    tokenizer = AutoTokenizer.from_pretrained("google/codegemma-2b")
+    model = AutoModelForCausalLM.from_pretrained('google/codegemma-2b', is_decoder=True)
+    #model = BertLMHeadModel.from_pretrained('google/codegemma-2b', is_decoder=True)
+    # Example assuming you have a prepared dataset for classification
+    #model = BertForSequenceClassification.from_pretrained('thenlper/gte-small', num_labels=2, is_decoder=True)  # binary classification
+    model.to(device)  # Move model to the appropriate device
 
-    # Load tokenizer and model
-    tokenizer = AutoTokenizer.from_pretrained("thenlper/gte-small")
-    model = AutoModelForCausalLM.from_pretrained("thenlper/gte-small")
-
-    # Train the model on your dataset
-    train_model(model, tokenizer, data)
+    train_model(model, tokenizer, data, device)
 
-    # Save the fine-tuned model
     model.save_pretrained("./fine_tuned_model")
     tokenizer.save_pretrained("./fine_tuned_model")
 
-    # Example usage
     prompt = "I need to retrieve the latest block on chain using a python script"
     api_query = generate_api_query(model, tokenizer, prompt, "latest block on chain", api_name, base_url)
     print(f"Generated code: {api_query}")
 
+def generate_api_query(model, tokenizer, prompt, desired_output, api_name, base_url):
+    # Prepare input prompt for the model, ensure tensors are compatible with PyTorch
+    input_ids = tokenizer.encode(f"{prompt} Write an API query to {api_name} to get {desired_output}", return_tensors="pt")
+
+    # Ensure input_ids are on the same device as the model
+    input_ids = input_ids.to(model.device)
+
+    # Generate query using model with temperature for randomness
+    output = model.generate(input_ids, max_length=256, temperature=0.1, do_sample=True)
+
+    # Decode the generated query tokens
+    query = tokenizer.decode(output[0], skip_special_tokens=True)
+    return f"{base_url}/{query}"
+
 if __name__ == "__main__":
     api_name = "Koios"
-    base_url = "https://api.koios.rest"
+    base_url = "https://api.koios.rest/v1"
     main(api_name, base_url)

logs/events.out.tfevents.1714322367.172-3-0-7.lightspeed.irvnca.sbcglobal.net.39122.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83827f2cf7d20a317b97a09a293ebac35eb1e809d395d2ec317c06950d3f40c6
+size 6596

results/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21b4ed4bb45f70522e361ac23b7d2e031a99706cbde4e236374a52b3d6b0b7a2
+size 133588624