Index

A

add_merges() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) add_token_suggestion() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

add_tokens() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

B

baseline_tokenizer (TokenizerChanger.tokenizer_changer.TokenizerChanger property) (TokenizerChanger.TokenizerChanger property)

D

delete_inappropriate_merges() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) delete_k_tail_tokens() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) delete_merges() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

delete_overlaps() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) delete_tokens() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

F

find_tail_tokens() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) find_token_id_gap() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

find_tokens() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) format_merges() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

G

get_overlapping_merges() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

get_overlapping_tokens() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

L

load_tokenizer() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

M

module TokenizerChanger TokenizerChanger.tokenizer_changer

N

none_keys (TokenizerChanger.tokenizer_changer.TokenizerChanger property) (TokenizerChanger.TokenizerChanger property)

R

replace_tokens() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

S

save_tokenizer() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) set_space_marker() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)

set_space_sign() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method) space_marker (TokenizerChanger.tokenizer_changer.TokenizerChanger property) (TokenizerChanger.TokenizerChanger property)

T

TokenizerChanger module TokenizerChanger (class in TokenizerChanger) (class in TokenizerChanger.tokenizer_changer)

TokenizerChanger.tokenizer_changer module tokens_to_delete (TokenizerChanger.tokenizer_changer.TokenizerChanger property) (TokenizerChanger.TokenizerChanger property)

U

updated_tokenizer() (TokenizerChanger.tokenizer_changer.TokenizerChanger method) (TokenizerChanger.TokenizerChanger method)