我想流一个大的.parquet文件，我在本地训练分类模型。我的脚本似乎只加载了第一个小批：即使文件非常大，时期的数量也会非常快地增加，1个时期应该持续大约10个小时。下面是我使用的代码：

import pyarrow.parquet as pq
import torch
import pandas as pd
import evaluate
from transformers import (AutoTokenizer, CamembertForSequenceClassification,
                          EarlyStoppingCallback, Trainer, TrainingArguments,
                          pipeline)
import numpy as np

class MyIterableDataset(torch.utils.data.IterableDataset):

    def __init__(self, parquet_file_path: str, tokenizer, label_encoder, batch_size: int = 8):
        self.parquet_file = pq.ParquetFile(parquet_file_path)
        self.generator = self.parquet_file.iter_batches(batch_size=batch_size)
        self.tokenizer = tokenizer
        self.label_encoder = label_encoder

    def __iter__(self):
        """ """
        data = next(self.generator)
        encodings = self.tokenizer(data['text'].tolist(), truncation=True, padding=True, max_length=512)
        items = []
        for idx in range(len(data)):
            item = {key: torch.tensor(val[idx]) for key, val in encodings.items()}
            item["labels"] = torch.tensor(self.label_encoder.transform([str(data['target'][idx])]))
            items.append(item)
        return iter(items)

个字符