1 
from joblib import Parallel, delayed
from collections import OrderedDict
from torchtext.data import  Dataset, Example, RawField, Field, NestedField

self.raw_content = RawField()
self.id = RawField()
self.raw_abstract = RawField(is_target=True)
self.content = NestedField(Field(fix_length=80), fix_length=50)
self.abstract = NestedField(Field())
self.abstract.is_target = True

self.fields = { "article": [("raw_content", self.raw_content) ("content", self.content)],
            "abstract": [ ("raw_abstract", self.raw_abstract)("abstract", self.abstract),],
            "id": [("id", self.id)]}


def load_fname(fname, reading_path, fields):
    fpath = os.path.join(reading_path, fname)
    with open(fpath, "r") as data:
        ex = Example.fromJSON(data.read(), fields)
    return (ex, fpath)

什么是Example.fromJSON(data.read(), fields), 但与 huggingface ( https://github.com/huggingface ) 的等价物？我需要通过一些变压器来改变机器学习模型中的一些 lstm。现在，要走的路是使用转换器对数据进行预处理。

编辑

>>> from datasets import load_dataset
>>> dataset = load_dataset('json', data_files='my_file.json', field='data')

来源：https ://huggingface.co/docs/datasets/loading_datasets.html

我想我将不得不使用上面的代码，但仍然不确定。

4

0 回答 0