def huggingface_from_pretrained(
    source: Union[Path, str], tok_config: Dict, trf_config: Dict
) -> HFObjects:
    """Create a Huggingface transformer model from pretrained weights. Will
    download the model if it is not already downloaded.
    source (Union[str, Path]): The name of the model or a path to it, such as
        'bert-base-cased'.
    tok_config (dict): Settings to pass to the tokenizer.
    trf_config (dict): Settings to pass to the transformer.
    """
    if hasattr(source, "absolute"):
        str_path = str(source.absolute())
    else:
        str_path = source
    tokenizer = AutoTokenizer.from_pretrained(str_path, **tok_config)
    vocab_file_contents = None
    if hasattr(tokenizer, "vocab_file"):
        with open(tokenizer.vocab_file, "rb") as fileh:
            vocab_file_contents = fileh.read()
    trf_config["return_dict"] = True
    config = AutoConfig.from_pretrained(str_path, **trf_config)
    transformer = AutoModel.from_pretrained(str_path, config=config)
    ops = get_current_ops()
    if isinstance(ops, CupyOps):
        transformer.cuda()
    return HFObjects(tokenizer, transformer, vocab_file_contents)

GiNZAのtransformers対応モデルであるja-ginza-electraではtokenizerとしてSudachiTraのElectraSudachipyTokenizerを用いていますが、sudachitraはtransformers標準のAuto Classesには登録されていないため、ただ単にElectraSudachipyTokenizerをtokenizer_config.jsonで指定するだけでは読み込んではくれません。spaCyではコードは基本的にコマンドラインからpython -m spacyで起動する形になるため、import sudachitraを実行することは困難です。

これを解決するために、GiNZAではginza-transformersというライブラリを作成し、上記のhugging_face_from_pretrained関数の機能を代替していますが、spacy-transformersに対して密結合な実装にならざるを得ず、メンテナンス性の低さが課題となっています。

AutoTokenizer で ElectraSudachipyTokenizer を読み込む

AutoTokenizer.register

Auto Classesのドキュメントではregister という新規クラスをAuto Classesに登録する関数が紹介されています。

from transformers import AutoConfig, AutoModel

AutoConfig.register("new-model", NewModelConfig)
AutoModel.register(NewModelConfig, NewModel)

"auto_map": {
    "AutoTokenizer": [
        "module.slow_tokenizer_class",
        "module.fast_tokenizer_class"
    ]
}

if use_fast and tokenizer_auto_map[1] is not None:
    class_ref = tokenizer_auto_map[1]
else:
    class_ref = tokenizer_auto_map[0]

module_file, class_name = class_ref.split(".")
tokenizer_class = get_class_from_dynamic_module(
    pretrained_model_name_or_path, module_file + ".py", class_name, **kwargs
)

revision (`str`, *optional*, defaults to `"main"`):
    The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
    git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
    identifier allowed by git.
trust_remote_code (`bool`, *optional*, defaults to `False`):
    Whether or not to allow for custom models defined on the Hub in their own modeling files. This option
    should only be set to `True` for repositories you trust and in which you have read the code, as it will
    execute code present on the Hub on your local machine.

# modeling.py
from sudachitra import ElectraSudachipyTokenizer

{
 "tokenizer_class": "ElectraSudachipyTokenizer",
 "do_lower_case": false,
 "do_word_tokenize": true,
 "do_subword_tokenize": true,
 "word_tokenizer_type": "sudachipy",
 "word_form_type": "dictionary_and_surface",
 "subword_tokenizer_type": "wordpiece",
 "model_max_length": 512,
 "sudachipy_kwargs": {"split_mode":"A","dict_type":"core"},
 "auto_map": {
    "AutoTokenizer": [
        "modeling.ElectraSudachipyTokenizer",
        null
    ]
 }
}

AutoTokenizer.from_pretrained("megagonlabs/electra-base-japanese-discriminator", trust_remote_code=True)