stuff/basic_function_test.py at main · ghostperpper007/stuff · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
from model import add_expert
from orchestrator import run_experts
from datasets import load_dataset
from training_manager import TrainingManager
from orchestrator import prepare_input, _load_tokenizer_and_embedding
output_dir = r"C:\model-weights"
descriptions = ["blibity bob", "corn on the cob", "grab a door knob", "tung sahere", "fishand ships", "fish and fries", "its called soccer", "du bist gut genug)",]
for i in range(7):
    add_expert(output_dir, descriptions[i])
    print("ze done")
print("running experts")
split = ['train']
ds = load_dataset("sahil2801/CodeAlpaca-20k", split=split, streaming=True)
ds = ds.take(10)
split=['validation']
eval = load_dataset("sahil2801/CodeAlpaca-20k", split=split, streaming = True)

for example in enumerate(ds):
    ds_input = example["instruction"]
    ds_output = example["output"]
    mdl_input = prepare_input(text=ds_input)
    mdl_output = run_experts(experts_dir=output_dir, text_input=ds_input)[3]