41 lines
1.4 KiB
Python
41 lines
1.4 KiB
Python
import numpy as np
|
|
import threading
|
|
import queue
|
|
|
|
def startDataWorker(dataset,encoding,batch_size,block_size):
|
|
data_q = queue.Queue(maxsize=100)
|
|
t = threading.Thread(target=dataWorker, args=(data_q, dataset, encoding, batch_size, block_size), daemon=True)
|
|
t.start()
|
|
while (1):
|
|
try:
|
|
bx, by = data_q.get(timeout=30)
|
|
except queue.Empty:
|
|
print("queue empty ...")
|
|
continue
|
|
yield (bx,by)
|
|
|
|
def dataWorker(q, dataset, encoding, batch_size, block_size):
|
|
batch_x, batch_y = [], []
|
|
while True:
|
|
for text in dataset["text"]:
|
|
tokens = encoding.encode(text)
|
|
for i in range(0, len(tokens)-block_size-1,block_size):
|
|
x = [encoding.bos_token_id] + tokens[i:i+block_size-1]
|
|
y = tokens[i:i+block_size]
|
|
|
|
if len(x) < block_size:
|
|
pad = len(x)-(block_size-1)
|
|
x = x + [encoding.eos_token_id] + [encoding.pad_token_id] * pad
|
|
|
|
if len(y) < block_size:
|
|
pad = len(y)-(block_size-1)
|
|
y = y + [encoding.eos_token_id] + [encoding.pad_token_id] * pad
|
|
|
|
batch_x.append(x)
|
|
batch_y.append(y)
|
|
|
|
if len(batch_x) == batch_size:
|
|
q.put((np.array(batch_x, dtype=np.int32),
|
|
np.array(batch_y, dtype=np.int32)))
|
|
batch_x, batch_y = [], []
|