This model is totally failed on unknown database.

#13
by mohitgoswami - opened

tokenizer = AutoTokenizer.from_pretrained("NumbersStation/nsql-llama-2-7B")
model = AutoModelForCausalLM.from_pretrained("NumbersStation/nsql-llama-2-7B", torch_dtype=torch.bfloat16,device_map='auto')
pipe = pipeline("text-generation",
model=model,
tokenizer= tokenizer,
torch_dtype=torch.bfloat16,
device_map="auto",
max_new_tokens = 500,
do_sample=True,
top_k=30,
num_return_sequences=1,
eos_token_id=tokenizer.eos_token_id
)
llm = HuggingFacePipeline(pipeline = pipe, model_kwargs = {'temperature':0})

text = """CREATE TABLE "Ball_by_Ball" (
"Match_Id" INTEGER NOT NULL,
"Over_Id" INTEGER NOT NULL,
"Ball_Id" INTEGER NOT NULL,
"Innings_No" INTEGER NOT NULL,
"Team_Batting" INTEGER NOT NULL,
"Team_Bowling" INTEGER NOT NULL,
"Striker_Batting_Position" INTEGER NOT NULL,
"Striker" INTEGER NOT NULL,
"Non_Striker" INTEGER NOT NULL,
"Bowler" INTEGER NOT NULL,
PRIMARY KEY ("Match_Id", "Over_Id", "Ball_Id", "Innings_No"),
FOREIGN KEY("Team_Bowling") REFERENCES "Team" ("Team_Id"),
FOREIGN KEY("Team_Batting") REFERENCES "Team" ("Team_Id"),
FOREIGN KEY("Bowler") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Non_Striker") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Striker") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Match" ("Match_Id")
);

CREATE TABLE "Batsman_Scored" (
"Match_Id" INTEGER NOT NULL,
"Over_Id" INTEGER NOT NULL,
"Ball_Id" INTEGER NOT NULL,
"Runs_Scored" INTEGER NOT NULL,
"Innings_No" INTEGER NOT NULL,
PRIMARY KEY ("Match_Id", "Over_Id", "Ball_Id", "Innings_No"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Ball_Id")
);

CREATE TABLE "Batting_Style" (
"Batting_Id" INTEGER NOT NULL,
"Batting_hand" VARCHAR(200) NOT NULL,
PRIMARY KEY ("Batting_Id")
);

CREATE TABLE "Bowling_Style" (
"Bowling_Id" INTEGER NOT NULL,
"Bowling_skill" VARCHAR(200) NOT NULL,
PRIMARY KEY ("Bowling_Id")
);

CREATE TABLE "City" (
"City_Id" INTEGER NOT NULL,
"City_Name" VARCHAR(200) NOT NULL,
"Country_id" INTEGER,
PRIMARY KEY ("City_Id"),
FOREIGN KEY("Country_id") REFERENCES "Country" ("Country_Id")
);

CREATE TABLE "Country" (
"Country_Id" INTEGER NOT NULL,
"Country_Name" VARCHAR(200) NOT NULL,
PRIMARY KEY ("Country_Id")
);

CREATE TABLE "Extra_Runs" (
"Match_Id" INTEGER NOT NULL,
"Over_Id" INTEGER NOT NULL,
"Ball_Id" INTEGER NOT NULL,
"Extra_Type_Id" INTEGER NOT NULL,
"Extra_Runs" INTEGER NOT NULL,
"Innings_No" INTEGER NOT NULL,
PRIMARY KEY ("Match_Id", "Over_Id", "Ball_Id", "Innings_No"),
FOREIGN KEY("Extra_Type_Id") REFERENCES "Extra_Type" ("Extra_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Ball_Id")
);

CREATE TABLE "Extra_Type" (
"Extra_Id" INTEGER NOT NULL,
"Extra_Name" VARCHAR(150) NOT NULL,
PRIMARY KEY ("Extra_Id")
);

CREATE TABLE "Match" (
"Match_Id" INTEGER NOT NULL,
"Team_1" INTEGER NOT NULL,
"Team_2" INTEGER NOT NULL,
"Match_Date" DATETIME NOT NULL,
"Season_Id" INTEGER NOT NULL,
"Venue_Id" INTEGER NOT NULL,
"Toss_Winner" INTEGER NOT NULL,
"Toss_Decide" INTEGER NOT NULL,
"Win_Type" INTEGER NOT NULL,
"Win_Margin" INTEGER,
"Outcome_type" INTEGER NOT NULL,
"Match_Winner" INTEGER,
"Man_of_the_Match" INTEGER,
PRIMARY KEY ("Match_Id"),
FOREIGN KEY("Win_Type") REFERENCES "Win_By" ("Win_Id"),
FOREIGN KEY("Venue_Id") REFERENCES "Venue" ("Venue_Id"),
FOREIGN KEY("Toss_Decide") REFERENCES "Toss_Decision" ("Toss_Id"),
FOREIGN KEY("Match_Winner") REFERENCES "Team" ("Team_Id"),
FOREIGN KEY("Toss_Winner") REFERENCES "Team" ("Team_Id"),
FOREIGN KEY("Team_2") REFERENCES "Team" ("Team_Id"),
FOREIGN KEY("Team_1") REFERENCES "Team" ("Team_Id"),
FOREIGN KEY("Season_Id") REFERENCES "Season" ("Season_Id"),
FOREIGN KEY("Man_of_the_Match") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Outcome_type") REFERENCES "Outcome" ("Outcome_Id")
);

CREATE TABLE "Out_Type" (
"Out_Id" INTEGER NOT NULL,
"Out_Name" VARCHAR(250) NOT NULL,
PRIMARY KEY ("Out_Id")
);

CREATE TABLE "Outcome" (
"Outcome_Id" INTEGER NOT NULL,
"Outcome_Type" VARCHAR(200) NOT NULL,
PRIMARY KEY ("Outcome_Id")
);

CREATE TABLE "Player" (
"Player_Id" INTEGER NOT NULL,
"Player_Name" VARCHAR(400) NOT NULL,
"DOB" DATETIME,
"Batting_hand" INTEGER NOT NULL,
"Bowling_skill" INTEGER,
"Country_Name" INTEGER NOT NULL,
PRIMARY KEY ("Player_Id"),
FOREIGN KEY("Country_Name") REFERENCES "Country" ("Country_Id"),
FOREIGN KEY("Bowling_skill") REFERENCES "Bowling_Style" ("Bowling_Id"),
FOREIGN KEY("Batting_hand") REFERENCES "Batting_Style" ("Batting_Id")
);

CREATE TABLE "Player_Match" (
"Match_Id" INTEGER NOT NULL,
"Player_Id" INTEGER NOT NULL,
"Role_Id" INTEGER NOT NULL,
"Team_Id" INTEGER,
PRIMARY KEY ("Match_Id", "Player_Id"),
FOREIGN KEY("Team_Id") REFERENCES "Team" ("Team_Id"),
FOREIGN KEY("Role_Id") REFERENCES "Rolee" ("Role_Id"),
FOREIGN KEY("Player_Id") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Match" ("Match_Id")
);

CREATE TABLE "Rolee" (
"Role_Id" INTEGER NOT NULL,
"Role_Desc" VARCHAR(150) NOT NULL,
PRIMARY KEY ("Role_Id")
);

CREATE TABLE "Season" (
"Season_Id" INTEGER NOT NULL,
"Man_of_the_Series" INTEGER NOT NULL,
"Orange_Cap" INTEGER NOT NULL,
"Purple_Cap" INTEGER NOT NULL,
"Season_Year" INTEGER,
PRIMARY KEY ("Season_Id"),
FOREIGN KEY("Purple_Cap") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Orange_Cap") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Man_of_the_Series") REFERENCES "Player" ("Player_Id")
);

CREATE TABLE "Team" (
"Team_Id" INTEGER NOT NULL,
"Team_Name" VARCHAR(450) NOT NULL,
PRIMARY KEY ("Team_Id")
);

CREATE TABLE "Toss_Decision" (
"Toss_Id" INTEGER NOT NULL,
"Toss_Name" VARCHAR(50) NOT NULL,
PRIMARY KEY ("Toss_Id")
);

CREATE TABLE "Umpire" (
"Umpire_Id" INTEGER NOT NULL,
"Umpire_Name" VARCHAR(350) NOT NULL,
"Umpire_Country" INTEGER NOT NULL,
PRIMARY KEY ("Umpire_Id"),
FOREIGN KEY("Umpire_Country") REFERENCES "Country" ("Country_Id")
);

CREATE TABLE "Venue" (
"Venue_Id" INTEGER NOT NULL,
"Venue_Name" VARCHAR(450) NOT NULL,
"City_Id" INTEGER,
PRIMARY KEY ("Venue_Id"),
FOREIGN KEY("City_Id") REFERENCES "City" ("City_Id")
);

CREATE TABLE "Wicket_Taken" (
"Match_Id" INTEGER NOT NULL,
"Over_Id" INTEGER NOT NULL,
"Ball_Id" INTEGER NOT NULL,
"Player_Out" INTEGER NOT NULL,
"Kind_Out" INTEGER NOT NULL,
"Fielders" INTEGER,
"Innings_No" INTEGER NOT NULL,
PRIMARY KEY ("Match_Id", "Over_Id", "Ball_Id", "Innings_No"),
FOREIGN KEY("Fielders") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Player_Out") REFERENCES "Player" ("Player_Id"),
FOREIGN KEY("Kind_Out") REFERENCES "Out_Type" ("Out_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Over_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Match_Id"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Innings_No"),
FOREIGN KEY("Innings_No") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Ball_Id") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Over_Id") REFERENCES "Ball_by_Ball" ("Ball_Id"),
FOREIGN KEY("Match_Id") REFERENCES "Ball_by_Ball" ("Ball_Id")
);

CREATE TABLE "Win_By" (
"Win_Id" INTEGER NOT NULL,
"Win_Type" VARCHAR(200) NOT NULL,
PRIMARY KEY ("Win_Id")
);

-- Using valid SQLite, answer the following questions for the tables provided above.

-- total number of teams

SELECT"""

input_ids = tokenizer(text, return_tensors="pt").input_ids# generated_ids = model.generate(input_ids, max_length=500)
input_ids = input_ids.to('cuda')
generated_ids = model.generate(input_ids, max_new_tokens=500)
query1=tokenizer.decode(generated_ids[0][len(input_ids[0]):], skip_special_tokens=True)
query = 'SELECT' +' '+ query1.split('SELECT')[-1]
query

result = 'SELECT id_id_no COMMENTITY_id_s_num_er_er_s_ COLL_id_no_no_ COLL_id_s_def_in_t_er_s_no_ COLL_no_ CO_id_s_no_ CO_u_n_er_s_ CO_no_ COLL_ CO_no_s_ CO_er_ CO_no_s_ CO_no_ CO_u_ CO_no_s_ CO_no_ CO_no_s_ CO_no_er_ CO_no_ CO_no_s_ CO_no_ CO_er_ CO_no_ CO_no_s_ CO_no_ CO_er_ CO_no_ CO_no_ CO_er_no_ CO_no_ CO_er_ CO_no_ CO_ CO_ CO_no_ CO_er_ CO_ CO_no_ CO_ CO_ CO_no_ CO_no_ CO_ CO_ CO_er_ CO_no_ CO_ CO_ CO_ CO_ CO_no_ CO_ CO_ CO_no_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_ CO_'

Its totally giving something else.

Is there any update on this task? bcoz i have a similar task

Hi,
I faced a similar issue recently. Just fine-tune with your dataset. It will run correctly.

Hey, could you help me with the fine-tuning code?

@AanVar , how many rows you had in you training dataset which you have used it for fine tuning?

@AanVar : Can you please share the sample of your dataset used in fine tuning ? Also did you use the same LORA config as mentioned in the github for fine tuning ?

Hi @Aanchal01 and @rusum ,
For finetuning my team has used LORA config. You can check out a code here for reference - https://www.linkedin.com/pulse/get-insight-from-your-business-data-build-llm-application-jain-2f

Our team created around 17k rows with single a column. An example of how we created the data:

Instruction: Create Table tablename (yourtablefields) -- Using SQLite, answer the following question from the given tables. -- yourquestion. ###Response: yourexpectedSQLquery

Even I am new to this field so still doing R&D.
Hope this helps.

Thanks @AanVar for the inputs...I am trying with the same approach, still not working as I am expecting...need to tweak around the LORA config parameters...

Sign up or log in to comment